You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@senssoft.apache.org by ar...@apache.org on 2016/11/20 14:16:26 UTC
[50/51] [partial] incubator-senssoft-tap git commit: Initial
containerized docker build for tap+es+distill
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/distill/models/brew.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/models/brew.py b/docker/distill/distill/models/brew.py
new file mode 100644
index 0000000..28d16b3
--- /dev/null
+++ b/docker/distill/distill/models/brew.py
@@ -0,0 +1,235 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from elasticsearch import Elasticsearch, TransportError
+from flask import jsonify
+from distill import es
+
+class Brew (object):
+ """
+ Distill supports basic CRUD operations and publishes the status
+ of an persistenct database. Eventually it will support ingesting logs sent from
+ an registered application.
+ """
+
+ @staticmethod
+ def get_status ():
+ """
+ Fetch the status of the underlying database instance.
+
+ :return: [bool] if connection to database instance has been established
+ """
+ return es.ping (ignore=[400, 404])
+
+ @staticmethod
+ def get_applications ():
+ """
+ Fetch all the registered applications in Distill.
+
+ .. note:: Private indexes starting with a period are not included in the result set
+
+ :return: [dict] dictionary of all registered applications and meta information
+ """
+ doc = {}
+ query = { "aggs" : {
+ "count_by_type" : {
+ "terms" : {
+ "field" : "_type",
+ "size" : 100
+ }
+ }
+ }
+ }
+
+ try:
+ cluster_status = es.cat.indices (h=["index"], pri=False)
+ x = cluster_status.splitlines()
+
+ for idx in x:
+ idx = idx.rstrip ()
+
+ # Ignore private indexes (like .kibana or .stout)
+ if idx [:1] != '.':
+ response = es.search (index=idx, body=query)
+ d = {}
+ for tag in response["aggregations"]["count_by_type"]["buckets"]:
+ d [tag ['key']] = tag ['doc_count']
+ doc [idx] = d
+ except TransportError as e:
+ doc ['error'] = e.info
+ except Exception as e:
+ doc ['error'] = str (e)
+ return doc
+
+ @staticmethod
+ def create (app):
+ """
+ Register a new application in Distill
+
+ .. code-block:: bash
+
+ {
+ "application" : "xdata_v3",
+ "health" : "green",
+ "num_docs" : 0,
+ "status" : "open"
+ }
+
+ :param app: [string] application name (e.g. xdata_v3)
+ :return: [dict] dictionary of application and its meta information
+ """
+
+ # ignore 400 cause by IndexAlreadyExistsException when creating an index
+ res = es.indices.create (index=app, ignore=[400, 404])
+ doc = _get_cluster_status (app)
+ return jsonify (doc)
+
+ @staticmethod
+ def read (app, app_type=None):
+ """
+ Fetch meta data associated with an application
+
+ .. code-block:: bash
+
+ Example:
+ {
+ "application" : "xdata_v3",
+ "health" : "green",
+ "num_docs" : "100",
+ "status" : "open"
+ "types" : {
+ "raw_logs" : {
+ "@timestamp" : "date",
+ "action" : "string",
+ "elementId" : "string"
+ },
+ "parsed" : {
+ "@timestamp" : "date",
+ "elementId_interval" : "string"
+ },
+ "graph" : {
+ "uniqueID" : "string",
+ "transition_count" : "long",
+ "p_value" : "float"
+ }
+ }
+ }
+
+ :param app: [string] application name (e.g. xdata_v3)
+ :return: [dict] dictionary of application and its meta information
+ """
+
+ return jsonify (_get_cluster_status (app, app_type=app_type))
+
+ @staticmethod
+ def update (app):
+ """
+ .. todo::
+ Currently not implemented
+ """
+
+ return jsonify (status="not implemented")
+
+ @staticmethod
+ def delete (app):
+ """
+ Technically closes the index so its content is not searchable.
+
+ .. code-block: bash
+
+ Example:
+ {
+ status: "Deleted index xdata_v3"
+ }
+
+ :param app: [string] application name (e.g. xdata_v3)
+ :return: [dict] status message of the event
+ """
+
+ es.indices.close (index=app, ignore=[400, 404])
+ return jsonify (status="Deleted index %s" % app)
+
+def _get_cluster_status (app, app_type=None):
+ """
+ Return cluster status, index health, and document count as string
+
+ @todo figure out how to count individual documents stored at an app_type (currently shows only index count)
+ :param app: [string] application name (e.g. xdata_v3)
+ :return: [dict] dictionary of index meta data including field names
+ """
+
+ doc = {}
+ try:
+ cluster_status = es.cat.indices (index=app, h=["health", "status", "docs.count"], pri=True, ignore=[400, 404])
+ v = str (cluster_status).split (" ")
+ m = ["health", "status", "num_docs"]
+ doc = dict (zip (m, v))
+ # Add back application
+ doc ["application"] = app
+ except TransportError as e:
+ doc ['error'] = e.info
+ except Exception as e:
+ doc ['error'] = str (e)
+
+ doc ['fields'] = _get_all_fields (app, app_type)
+ return doc
+
+def _parse_mappings (app, app_type=None):
+ """
+ .. todo:
+
+ Need to parse out result set that presents field list and type
+ """
+
+ try:
+ mappings = es.indices.get_mapping (index=app, doc_type=[app_type], ignore=[400, 404])
+ # mappings = yaml.safe_load (json.ess (mappings))
+ # print json.dumps (mappings [app]["mappings"], indent=4, separators=(',', ': '))
+ ignore = ["properties", "format"]
+ except TransportError as e:
+ doc ['error'] = e.info
+ except Exception as e:
+ doc ['error'] = str (e)
+ return doc
+
+def _get_all_fields (app, app_type=None):
+ """
+ Retrieve all possible fields in an application
+
+ :param app: [string] application name (e.g. xdata_v3)
+ :param app_type: [string] application type (e.g. logs)
+ :return: [list] list of strings representing the fields names
+ """
+ d = list ()
+ query = { "aggs" : {
+ "fields" : {
+ "terms" : {
+ "field" : "_field_names",
+ "size" : 100
+ }
+ }
+ }
+ }
+
+ try:
+ response = es.search (index=app, doc_type=app_type, body=query)
+ for tag in response['aggregations']['fields']['buckets']:
+ d.append (tag ['key'])
+ except TransportError as e:
+ d.append (str (e.info))
+ except Exception as e:
+ d.append (str (e))
+ return d
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/distill/models/stout.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/models/stout.py b/docker/distill/distill/models/stout.py
new file mode 100644
index 0000000..d6421d8
--- /dev/null
+++ b/docker/distill/distill/models/stout.py
@@ -0,0 +1,149 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from distill import app, es
+from elasticsearch_dsl import DocType, String, Boolean, Date, Nested, Search
+from elasticsearch_dsl.query import MultiMatch, Match, Q
+from elasticsearch import Elasticsearch, TransportError
+from flask import jsonify
+import pandas as pd
+
+class StoutDoc (DocType):
+ """
+ Representation of a Stout documentat.
+ """
+
+ sessionID = String (index="not_analyzed")
+ task1 = Nested ()
+ task2 = Nested ()
+
+ class Meta:
+ index = '.stout'
+ doc_type = 'testing'
+
+ def save (self, *args, **kwargs):
+ """
+ Save data from parsing as a Stout document in Distill
+ """
+ return super (StoutDoc, self).save (*args, **kwargs)
+
+class Stout (object):
+ """
+ Main Stout class to support ingest and search operations.
+ """
+
+ @staticmethod
+ def ingest ():
+ """
+ Ingest data coming from Stout to Distill
+ """
+
+ # Create the mappings in elasticsearch
+ StoutDoc.init ()
+ status = True
+ data = _parse ();
+ try:
+ for k,v in data.items ():
+ doc = StoutDoc ()
+ if 'sessionID' in v:
+ doc.sessionID = v['sessionID']
+ if 'task1' in v:
+ doc.task1 = v['task1']
+ if 'task2' in v:
+ doc.task2 = v['task2']
+ doc.save ()
+ except Error as e:
+ status = False
+ return jsonify (status=status)
+
+def _parse ():
+ """
+ Parse master answer table with mapping into an associative array
+
+ :return: [dict] dictionary of session information
+ """
+ master = app.config ['MASTER']
+ mappings = app.config ['MAPPINGS']
+
+ fileContents=pd.read_csv(master, encoding='utf-8')
+ plainTextMappings=pd.read_csv(mappings, encoding='raw_unicode_escape')
+ headers=list(fileContents.columns.values)
+
+ #generate the mapping between header and plain text
+ translationRow={};
+ for fieldIndex in range(1,len(headers)):
+ t=plainTextMappings.ix[fieldIndex]
+ translationRow[headers[fieldIndex]]=t[9]
+
+ dictBySessionID={}
+ translationRow['items.text']='foo'
+ index=0
+ for row in fileContents.iterrows():
+ index=index+1
+
+ taskMetrics={}
+ index,data=row
+ identifier=row[1][0].split("::")
+ sessionID=identifier[0]
+ taskID=(identifier[1])
+ workingData={}
+ #is this session id already in the dictionary?
+ if sessionID in dictBySessionID:
+ #grab the entry as workingData
+ workingData=dictBySessionID[sessionID]
+
+ sysData={}
+ task1Data={}
+ task2Data={}
+ metaData={}
+ d={}
+
+ for fieldIndex in range(1,len(headers)):
+ if not pd.isnull(row[1][fieldIndex]): #only interested in non-null fields
+ tempDict={}
+ if headers[fieldIndex] in translationRow:
+ tempDict['field']=translationRow[headers[fieldIndex]]
+ #tempDict['field']=translationRow[9]
+ tempDict['value']=row[1][fieldIndex]
+ d[headers[fieldIndex]]=row[1][fieldIndex]
+ if "SYS" in headers[fieldIndex]:
+ sysData[headers[fieldIndex]]=tempDict
+ elif "OT1" in headers[fieldIndex]:
+ task1Data[headers[fieldIndex]]=tempDict
+ elif "OT2" in headers[fieldIndex]:
+ task2Data[headers[fieldIndex]]=tempDict
+ else:
+ metaData[headers[fieldIndex]]=tempDict
+
+ if d['TSK_TIME_DIFF_']>0: #block tasks with zero time elapsed
+ a=int(d['TSK_TIME_DIFF_OT1_'])
+ b=int(d['TSK_TIME_DIFF_OT2_'])
+ #figure out which task the values belong to
+ if ((a>0) & (b<=0)):
+ task1Data['taskID']=taskID
+ task1Data['meta']=metaData
+ task1Data['system']=sysData
+ workingData['task1']=task1Data
+ elif ((a<=0) & (b>0)):
+ task2Data['taskID']=taskID
+ task2Data['meta']=metaData
+ task2Data['system']=sysData
+ workingData['task2']=task2Data
+ else:
+ raise ValueError('Encountered an unexpected task time diff state')
+
+ workingData['sessionID'] = sessionID
+ dictBySessionID[sessionID]=workingData
+ return dictBySessionID
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/distill/models/tests/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/models/tests/__init__.py b/docker/distill/distill/models/tests/__init__.py
new file mode 100644
index 0000000..f6f6899
--- /dev/null
+++ b/docker/distill/distill/models/tests/__init__.py
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+distill: tests module.
+
+Meant for use with py.test.
+Organize tests into files, each named xxx_test.py
+Read more here: http://pytest.org/
+'''
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/distill/models/userale.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/models/userale.py b/docker/distill/distill/models/userale.py
new file mode 100644
index 0000000..f63fa51
--- /dev/null
+++ b/docker/distill/distill/models/userale.py
@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from elasticsearch import Elasticsearch, TransportError
+from elasticsearch_dsl import DocType, String, Boolean, Date, Float, Search
+from elasticsearch_dsl.query import MultiMatch, Match, Q
+from elasticsearch import Elasticsearch, TransportError
+from elasticsearch_dsl.connections import connections
+from werkzeug.datastructures import ImmutableMultiDict, MultiDict
+
+from flask import jsonify, Markup
+from distill import app, es
+import datetime
+
+class UserAle (object):
+ """
+ Main method of entry to perform segmentation and integration of STOUT's master
+ answer table (if STOUT is enabled). Advanced and basic analytics is performed in the
+ distill.algorithms.stats and distill.algorithms.graphs module.
+ """
+
+ @staticmethod
+ def segment (app, app_type=None, params=''):
+ """
+ Just support match all for now.
+ """
+ q = params.get ("q") if params.get ("q") else {}
+ fields = params.get ("fields") if params.get ("fields") else []
+ size = params.get ("size") if params.get ("size") else 10
+ scroll = params.get ("scroll") if params.get ("scroll") else False
+ fl = params.get ("fl") if params.get ("fl") else []
+
+ # filters = params.get ("filter") if params.get ("filter") else {}
+
+ # 'q': args.get('q', '{}'),
+ # 'fields': args.get('fl', '{}'),
+ # 'size': args.get ('size', 100),
+ # 'scroll': args.get ('scroll', False),
+ # 'filters': request_args.getlist ('fq')
+ query = {}
+ query ['size'] = size
+
+ if q:
+ res = q.split(":")
+ key = res [0]
+ val = res [1]
+ query ['query'] = {"match" : { key : val } }
+ else:
+ query ['query'] = {"match_all" : {}}
+
+ if len (fields) > 0:
+ ex = {
+ "include" : fields.split(",")
+ }
+ query ['_source'] = ex
+
+
+ response = es.search (index=app, doc_type=app_type, body=query)
+
+ return jsonify (response)
+
+ @staticmethod
+ def search (app,
+ app_type=None,
+ filters=list (),
+ size=100,
+ include="*",
+ scroll=None,
+ sort_field=None):
+ """
+ Perform a search query.
+
+ :param app: [string] application id (e.g. "xdata_v3")
+ :param app_type: [string] name of the application type. If None all application types are searched.
+ :param filters: [list of strings] list of filters for a query.
+ :param size: [int] maximum number of hits that should be returned
+ :param sort_field: [string] sorting field. Currently supported fields: "timestamp", "date"
+ :return: [dict] dictionary with processed results. If STOUT is enabled, STOUT data will be merged with final result.
+ """
+
+ # Need some query builder...
+ log_result = es.search (index=app, doc_type=app_type, body=query, fields=filters, size=size)
+
+ stout_result = Stout.getSessions ()
+
+ data = merged_results (log_result, stout_result)
+ return data
+
+ @staticmethod
+ def denoise (app, app_type='parsed', save=False):
+ """
+ """
+ pass
+
+"""
+Combine a list of dictionaries together to form one complete dictionary
+"""
+def merge_dicts (lst):
+ dall = {}
+ for d in lst:
+ dall.update (d)
+ return dall
+
+"""
+Get query parameters from the request and preprocess them.
+:param [dict-like structure] Any structure supporting get calls
+:result [dict] Parsed parameters
+"""
+def parse_query_parameters (indx, app_type=None, request_args = {}):
+ args = {key: value[0] for (key, value) in dict (request_args).iteritems ()}
+
+ # print "args = ", args
+ # Parse out simple filter queries
+ filters = []
+ for filter in get_all_fields (indx, app_type):
+ if filter in args:
+ filters.append((filter, args[filter]))
+
+ return {
+ 'q': args.get('q', '{}'),
+ 'fields': args.get('fl', []),
+ 'size': args.get ('size', 100),
+ 'scroll': args.get ('scroll', False),
+ 'filters': request_args.getlist ('fq')
+ }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/distill/server.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/server.py b/docker/distill/distill/server.py
new file mode 100644
index 0000000..23acd83
--- /dev/null
+++ b/docker/distill/distill/server.py
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from distill import app
+from distill.app import *
+
+"""
+Start up a local WSGI server called development
+"""
+def dev_server ():
+ host = app.config ['HOST']
+ port = app.config ['PORT']
+ debug = app.config ['DEBUG']
+ app.run (host=host, port=port, debug=debug)
+
+if __name__ == '__main__':
+ dev_server ()
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/distill/tests/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/tests/__init__.py b/docker/distill/distill/tests/__init__.py
new file mode 100644
index 0000000..09c5e2f
--- /dev/null
+++ b/docker/distill/distill/tests/__init__.py
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''
+distill: tests module.
+
+Meant for use with py.test.
+Organize tests into files, each named xxx_test.py
+Read more here: http://pytest.org/
+'''
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/distill/tests/basic_test.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/tests/basic_test.py b/docker/distill/distill/tests/basic_test.py
new file mode 100644
index 0000000..712d1fe
--- /dev/null
+++ b/docker/distill/distill/tests/basic_test.py
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''
+distill: Test module.
+
+Meant for use with py.test.
+Write each test as a function named test_<something>.
+Read more here: http://pytest.org/
+'''
+
+def test_example():
+ assert True
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/distill/tests/distill_test.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/tests/distill_test.py b/docker/distill/distill/tests/distill_test.py
new file mode 100644
index 0000000..2fb6502
--- /dev/null
+++ b/docker/distill/distill/tests/distill_test.py
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from flask import Flask, request
+
+from distill import app as test_app
+
+def test_example ():
+ assert True
+ # with test_app.test_client () as c:
+ # rv = c.get ('/?tequila=42')
+ # assert request.args ['tequila'] == '42'
+
+# import os
+# import flaskr
+# import unittest
+# import tempfile
+
+# class FlaskrTestCase(unittest.TestCase):
+
+# def setUp(self):
+# self.db_fd, flaskr.app.config['DATABASE'] = tempfile.mkstemp()
+# flaskr.app.config['TESTING'] = True
+# self.app = flaskr.app.test_client()
+# flaskr.init_db()
+
+# def tearDown(self):
+# os.close(self.db_fd)
+# os.unlink(flaskr.app.config['DATABASE'])
+
+# if __name__ == '__main__':
+# unittest.main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/distill/utils/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/utils/__init__.py b/docker/distill/distill/utils/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/distill/utils/exceptions.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/utils/exceptions.py b/docker/distill/distill/utils/exceptions.py
new file mode 100644
index 0000000..a391241
--- /dev/null
+++ b/docker/distill/distill/utils/exceptions.py
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class Error (Exception):
+ """Base class for exceptions."""
+ pass
+
+class ValidationError (Error):
+ """ Exceptions raised for errors in validated a url."""
+
+ def __init__ (self, url, msg):
+ self.url = url
+ self.msg = msg
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/distill/utils/query_builder.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/utils/query_builder.py b/docker/distill/distill/utils/query_builder.py
new file mode 100644
index 0000000..017a08b
--- /dev/null
+++ b/docker/distill/distill/utils/query_builder.py
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class QueryBuilder (object):
+
+ def __init__ (self, query=None):
+ if query:
+ self.query = query
+ else:
+ self.query = {
+ "query" : {
+ "match_all" : {}
+ }
+ }
+
+
+ def add_filters (self, filters):
+ pass
+
+ def add_sorting (self, sort_field='', sort_order=''):
+ pass
+
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/distill/utils/tests/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/utils/tests/__init__.py b/docker/distill/distill/utils/tests/__init__.py
new file mode 100644
index 0000000..09c5e2f
--- /dev/null
+++ b/docker/distill/distill/utils/tests/__init__.py
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''
+distill: tests module.
+
+Meant for use with py.test.
+Organize tests into files, each named xxx_test.py
+Read more here: http://pytest.org/
+'''
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/distill/utils/validation.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/utils/validation.py b/docker/distill/distill/utils/validation.py
new file mode 100644
index 0000000..7cd3362
--- /dev/null
+++ b/docker/distill/distill/utils/validation.py
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from distill.utils.exceptions import ValidationError
+
+def validate_request (q):
+ """
+ Parse out request message and validate inputs
+
+ :param q: Url query string
+ :raises ValidationError: if the query is missing required parameters
+ """
+ if 'q' not in q:
+ raise ValidationError ("Missing required parameter: %s" % 'q')
+ else:
+ # Handle rest of parsing
+ pass
+
+def str2bool (v):
+ """
+ Convert string expression to boolean
+
+ :param v: Input value
+ :returns: Converted message as boolean type
+ :rtype: bool
+ """
+ return v.lower() in ("yes", "true", "t", "1")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/distill/version.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/version.py b/docker/distill/distill/version.py
new file mode 100644
index 0000000..6532ea7
--- /dev/null
+++ b/docker/distill/distill/version.py
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Version information for Distill.
+
+This file is imported by ``Distill.__init__``,
+and parsed by ``setup.py``.
+"""
+
+__version__ = "0.1.3"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/requirements.txt
----------------------------------------------------------------------
diff --git a/docker/distill/requirements.txt b/docker/distill/requirements.txt
new file mode 100644
index 0000000..5137ad9
--- /dev/null
+++ b/docker/distill/requirements.txt
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Flask==0.10.1
+#networkx==1.11
+elasticsearch-dsl==2.0.0
+#numpy>=1.10.0
+#scipy>=0.17.0
+pandas>=0.18.1
+pytest>=3.0.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/setup.cfg
----------------------------------------------------------------------
diff --git a/docker/distill/setup.cfg b/docker/distill/setup.cfg
new file mode 100644
index 0000000..08020f4
--- /dev/null
+++ b/docker/distill/setup.cfg
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[egg_info]
+tag_build = 0.1.4
+tag_svn_revision = false
+
+[aliases]
+test=pytest
+
+[tool:pytest]
+addopts = --verbose --ignore=build --ignore=setup.py --ignore=dist --junitxml=test-report.xml --cov-report xml --cov=distill distill/.
+norecursedirs = *.eggs *env* .git
+
+[build_sphinx]
+source-dir = docs
+build-dir = docs/_build
+all_files = 1
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/distill/setup.py
----------------------------------------------------------------------
diff --git a/docker/distill/setup.py b/docker/distill/setup.py
new file mode 100644
index 0000000..8ddd32f
--- /dev/null
+++ b/docker/distill/setup.py
@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from setuptools import setup, find_packages
+import distutils.cmd
+import distutils.log
+from setuptools.command.test import test as TestCommand
+import io, os, sys, subprocess
+
+if sys.version_info[:2] < (2, 7):
+ m = "Python 2.7 or later is required for Distill (%d.%d detected)."
+ raise ImportError (m % sys.version_info[:2])
+
+if sys.argv[-1] == 'setup.py':
+ print ("To install, run 'python setup.py install'")
+ print ()
+
+def read (*filenames, **kwargs):
+ encoding = kwargs.get ('encoding', 'utf-8')
+ sep = kwargs.get ('sep', '\n')
+ buf = []
+ for filename in filenames:
+ with io.open (filename, encoding=encoding) as f:
+ buf.append (f.read ())
+ return sep.join (buf)
+
+# Get the version string
+def get_version ():
+ basedir = os.path.dirname (__file__)
+ with open (os.path.join (basedir, 'distill/version.py')) as f:
+ version = {}
+ exec (f.read (), version)
+ return version['__version__']
+ raise RuntimeError ('No version info found.')
+
+setup (
+ name = "Distill",
+ version = get_version (),
+ url = "https://github.com/apache/incubator-senssoft-distill",
+ license = "Apache Software License",
+ author = "Michelle Beard",
+ author_email = "msbeard@apache.org",
+ description = "An analytical framework for UserALE.",
+ long_description = __doc__,
+ classifiers = [
+ 'Development Status :: 4 - Beta',
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 2.7',
+ 'Natural Language :: English',
+ 'Environment :: Web Environment',
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: Apache Software License',
+ 'Operating System :: OS Independent',
+ 'Private :: Do Not Upload"'
+ ],
+ keywords = "stout userale tap distill", # Separate with spaces
+ packages = find_packages (exclude=['examples', 'tests']),
+ include_package_data = True,
+ zip_safe = False,
+ setup_requires = ['pytest-runner'],
+ tests_require = ['pytest>=3.0.0', 'pytest-pylint', 'coverage'],
+ install_requires = ['Flask==0.10.1',
+ #'networkx==1.11',
+ 'elasticsearch-dsl==2.0.0',
+ #'numpy>=1.10.0',
+ #'scipy>=0.17.0',
+ 'pandas>=0.18.1'
+ ],
+ entry_points = {
+ 'console_scripts': [
+ 'dev = distill.server:dev_server'
+ ]
+ }
+)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/es/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/es/Dockerfile b/docker/es/Dockerfile
new file mode 100644
index 0000000..b84b637
--- /dev/null
+++ b/docker/es/Dockerfile
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Ubuntu Xenial 16.04 [LTS]
+FROM ubuntu:16.04
+
+# install system wide deps
+RUN apt-get -yqq update
+RUN apt-get -yqq install openjdk-8-jre
+
+RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4
+
+ENV ELASTICSEARCH_VERSION 2.3.5
+ENV ELASTICSEARCH_REPO_BASE http://packages.elasticsearch.org/elasticsearch/2.x/debian
+
+RUN echo "deb $ELASTICSEARCH_REPO_BASE stable main" > /etc/apt/sources.list.d/elasticsearch.list
+
+# install elasticsearch
+RUN set -x \
+ && apt-get -yqq update \
+ && apt-get -yqq install --no-install-recommends elasticsearch=$ELASTICSEARCH_VERSION \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV PATH /usr/share/elasticsearch/bin:$PATH
+
+WORKDIR /usr/share/elasticsearch
+
+RUN set -ex \
+ && for path in \
+ ./data \
+ ./logs \
+ ./config \
+ ./config/scripts \
+ ; do \
+ mkdir -p "$path"; \
+ chown -R elasticsearch:elasticsearch "$path"; \
+ done
+
+COPY elasticsearch.yml /usr/share/elasticsearch/config
+COPY logging.yml /usr/share/elasticsearch/config
+
+USER elasticsearch
+
+# Install Elastic-HQ
+RUN plugin install royrusso/elasticsearch-HQ/2.0.3
+
+CMD ["elasticsearch"]
+
+# Run on ports 9200 & 9300
+EXPOSE 9200 9300
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/es/elasticsearch.yml
----------------------------------------------------------------------
diff --git a/docker/es/elasticsearch.yml b/docker/es/elasticsearch.yml
new file mode 100644
index 0000000..6126feb
--- /dev/null
+++ b/docker/es/elasticsearch.yml
@@ -0,0 +1,94 @@
+# ======================== Elasticsearch Configuration =========================
+#
+# NOTE: Elasticsearch comes with reasonable defaults for most settings.
+# Before you set out to tweak and tune the configuration, make sure you
+# understand what are you trying to accomplish and the consequences.
+#
+# The primary way of configuring a node is via this file. This template lists
+# the most important settings you may want to configure for a production cluster.
+#
+# Please see the documentation for further information on configuration options:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration.html>
+#
+# ---------------------------------- Cluster -----------------------------------
+#
+# Use a descriptive name for your cluster:
+#
+cluster.name: SensSoft
+#
+# ------------------------------------ Node ------------------------------------
+#
+# Use a descriptive name for the node:
+#
+node.name: soft-01
+#
+# Add custom attributes to the node:
+#
+# node.rack: r1
+#
+# ----------------------------------- Paths ------------------------------------
+#
+# Path to directory where to store the data (separate multiple locations by comma):
+#
+path.data: /usr/share/elasticsearch/data
+#
+# Path to log files:
+#
+path.logs: /usr/share/elasticsearch/logs
+#
+# ----------------------------------- Memory -----------------------------------
+#
+# Lock the memory on startup:
+#
+# bootstrap.mlockall: true
+#
+# Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory
+# available on the system and that the owner of the process is allowed to use this limit.
+#
+# Elasticsearch performs poorly when the system is swapping the memory.
+#
+# ---------------------------------- Network -----------------------------------
+#
+# Set the bind address to a specific IP (IPv4 or IPv6):
+#
+network.host: 0.0.0.0
+#
+# Set a custom port for HTTP:
+#
+# http.port: 9200
+#
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html>
+#
+# --------------------------------- Discovery ----------------------------------
+#
+# Pass an initial list of hosts to perform discovery when new node is started:
+# The default list of hosts is ["127.0.0.1", "[::1]"]
+#
+# discovery.zen.ping.unicast.hosts: ["host1", "host2"]
+#
+# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
+#
+# discovery.zen.minimum_master_nodes: 3
+#
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html>
+#
+# ---------------------------------- Gateway -----------------------------------
+#
+# Block initial recovery after a full cluster restart until N nodes are started:
+#
+# gateway.recover_after_nodes: 3
+#
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html>
+#
+# ---------------------------------- Various -----------------------------------
+#
+# Disable starting multiple nodes on a single system:
+#
+# node.max_local_storage_nodes: 1
+#
+# Require explicit names when deleting indices:
+#
+# action.destructive_requires_name: true
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/es/logging.yml
----------------------------------------------------------------------
diff --git a/docker/es/logging.yml b/docker/es/logging.yml
new file mode 100644
index 0000000..939aa1e
--- /dev/null
+++ b/docker/es/logging.yml
@@ -0,0 +1,85 @@
+# you can override this using by setting a system property, for example -Des.logger.level=DEBUG
+es.logger.level: INFO
+rootLogger: ${es.logger.level}, console, file
+logger:
+ # log action execution errors for easier debugging
+ action: DEBUG
+
+ # deprecation logging, turn to DEBUG to see them
+ deprecation: INFO, deprecation_log_file
+
+ # reduce the logging for aws, too much is logged under the default INFO
+ com.amazonaws: WARN
+ # aws will try to do some sketchy JMX stuff, but its not needed.
+ com.amazonaws.jmx.SdkMBeanRegistrySupport: ERROR
+ com.amazonaws.metrics.AwsSdkMetrics: ERROR
+
+ org.apache.http: INFO
+
+ # gateway
+ #gateway: DEBUG
+ #index.gateway: DEBUG
+
+ # peer shard recovery
+ #indices.recovery: DEBUG
+
+ # discovery
+ #discovery: TRACE
+
+ index.search.slowlog: TRACE, index_search_slow_log_file
+ index.indexing.slowlog: TRACE, index_indexing_slow_log_file
+
+additivity:
+ index.search.slowlog: false
+ index.indexing.slowlog: false
+ deprecation: false
+
+appender:
+ console:
+ type: console
+ layout:
+ type: consolePattern
+ conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+ file:
+ type: dailyRollingFile
+ file: ${path.logs}/${cluster.name}.log
+ datePattern: "'.'yyyy-MM-dd"
+ layout:
+ type: pattern
+ conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %.10000m%n"
+
+ # Use the following log4j-extras RollingFileAppender to enable gzip compression of log files.
+ # For more information see https://logging.apache.org/log4j/extras/apidocs/org/apache/log4j/rolling/RollingFileAppender.html
+ #file:
+ #type: extrasRollingFile
+ #file: ${path.logs}/${cluster.name}.log
+ #rollingPolicy: timeBased
+ #rollingPolicy.FileNamePattern: ${path.logs}/${cluster.name}.log.%d{yyyy-MM-dd}.gz
+ #layout:
+ #type: pattern
+ #conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+ deprecation_log_file:
+ type: dailyRollingFile
+ file: ${path.logs}/${cluster.name}_deprecation.log
+ datePattern: "'.'yyyy-MM-dd"
+ layout:
+ type: pattern
+ conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+ index_search_slow_log_file:
+ type: dailyRollingFile
+ file: ${path.logs}/${cluster.name}_index_search_slowlog.log
+ datePattern: "'.'yyyy-MM-dd"
+ layout:
+ type: pattern
+ conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+ index_indexing_slow_log_file:
+ type: dailyRollingFile
+ file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log
+ datePattern: "'.'yyyy-MM-dd"
+ layout:
+ type: pattern
+ conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/kibana/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile
new file mode 100644
index 0000000..e37cfa0
--- /dev/null
+++ b/docker/kibana/Dockerfile
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Ubuntu Xenial 16.04 [LTS]
+FROM ubuntu:16.04
+
+# install system wide deps
+RUN apt-get -yqq update
+RUN apt-get -yqq install netcat
+
+RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4
+
+ENV KIBANA_VERSION 4.5.4
+ENV KIBANA_REPO_BASE http://packages.elasticsearch.org/kibana/4.5/debian
+
+RUN echo "deb $KIBANA_REPO_BASE stable main" > /etc/apt/sources.list.d/kibana.list
+
+# install kibana
+RUN set -x \
+ && apt-get -yqq update \
+ && apt-get -yqq install --no-install-recommends kibana=$KIBANA_VERSION \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV PATH /opt/kibana/bin:$PATH
+
+WORKDIR /opt/kibana
+
+RUN set -ex \
+ && for path in \
+ ./config \
+ ; do \
+ chown -R kibana:kibana "$path"; \
+ done
+
+COPY kibana.yml /opt/kibana/config
+
+COPY entrypoint.sh /tmp/entrypoint.sh
+RUN chmod +x /tmp/entrypoint.sh
+
+# Install Sense plugin
+RUN kibana plugin --install elastic/sense
+
+USER kibana
+
+CMD ["/tmp/entrypoint.sh"]
+
+# Run on port 5601
+EXPOSE 5601
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/kibana/entrypoint.sh
----------------------------------------------------------------------
diff --git a/docker/kibana/entrypoint.sh b/docker/kibana/entrypoint.sh
new file mode 100644
index 0000000..02bd91f
--- /dev/null
+++ b/docker/kibana/entrypoint.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Wait for the Elasticsearch container to be ready before starting Kibana.
+echo "Stalling for Elasticsearch"
+while true; do
+ nc -q 1 elasticsearch 9200 2>/dev/null && break
+done
+
+echo "Starting Kibana"
+exec kibana
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/kibana/kibana.yml
----------------------------------------------------------------------
diff --git a/docker/kibana/kibana.yml b/docker/kibana/kibana.yml
new file mode 100644
index 0000000..8bc978f
--- /dev/null
+++ b/docker/kibana/kibana.yml
@@ -0,0 +1,79 @@
+# Kibana is served by a back end server. This controls which port to use.
+server.port: 5601
+
+# The host to bind the server to.
+server.host: "0.0.0.0"
+
+# If you are running kibana behind a proxy, and want to mount it at a path,
+# specify that path here. The basePath can't end in a slash.
+# server.basePath: ""
+
+# The maximum payload size in bytes on incoming server requests.
+# server.maxPayloadBytes: 1048576
+
+# The Elasticsearch instance to use for all your queries.
+elasticsearch.url: "http://elasticsearch:9200"
+
+# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false,
+# then the host you use to connect to *this* Kibana instance will be sent.
+elasticsearch.preserveHost: true
+
+# Kibana uses an index in Elasticsearch to store saved searches, visualizations
+# and dashboards. It will create a new index if it doesn't already exist.
+kibana.index: ".kibana"
+
+# The default application to load.
+kibana.defaultAppId: "discover"
+
+# If your Elasticsearch is protected with basic auth, these are the user credentials
+# used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana
+# users will still need to authenticate with Elasticsearch (which is proxied through
+# the Kibana server)
+# elasticsearch.username: "user"
+# elasticsearch.password: "pass"
+
+# SSL for outgoing requests from the Kibana Server to the browser (PEM formatted)
+# server.ssl.cert: /path/to/your/server.crt
+# server.ssl.key: /path/to/your/server.key
+
+# Optional setting to validate that your Elasticsearch backend uses the same key files (PEM formatted)
+# elasticsearch.ssl.cert: /path/to/your/client.crt
+# elasticsearch.ssl.key: /path/to/your/client.key
+
+# If you need to provide a CA certificate for your Elasticsearch instance, put
+# the path of the pem file here.
+# elasticsearch.ssl.ca: /path/to/your/CA.pem
+
+# Set to false to have a complete disregard for the validity of the SSL
+# certificate.
+# elasticsearch.ssl.verify: true
+
+# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to
+# request_timeout setting
+# elasticsearch.pingTimeout: 1500
+
+# Time in milliseconds to wait for responses from the back end or elasticsearch.
+# This must be > 0
+# elasticsearch.requestTimeout: 30000
+
+# Time in milliseconds for Elasticsearch to wait for responses from shards.
+# Set to 0 to disable.
+# elasticsearch.shardTimeout: 0
+
+# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying
+# elasticsearch.startupTimeout: 5000
+
+# Set the path to where you would like the process id file to be created.
+# pid.file: /var/run/kibana.pid
+
+# If you would like to send the log output to a file you can set the path below.
+logging.dest: stdout
+
+# Set this to true to suppress all logging output.
+# logging.silent: false
+
+# Set this to true to suppress all logging output except for error messages.
+logging.quiet: true
+
+# Set this to true to log all events, including system usage information and all requests.
+# logging.verbose: false
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/logstash/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/logstash/Dockerfile b/docker/logstash/Dockerfile
new file mode 100644
index 0000000..bd5929b
--- /dev/null
+++ b/docker/logstash/Dockerfile
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Ubuntu Xenial 16.04 [LTS]
+FROM ubuntu:16.04
+
+# install system wide deps
+RUN apt-get -yqq update
+RUN apt-get -yqq install openjdk-8-jre
+RUN apt-get -yqq install wget
+RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4
+
+ENV LOGSTASH_VERSION 1:2.3.4-1
+ENV LOGSTASH_REPO_BASE http://packages.elasticsearch.org/logstash/2.3/debian
+
+RUN echo "deb $LOGSTASH_REPO_BASE stable main" > /etc/apt/sources.list.d/logstash.list
+
+# install logstash
+RUN set -x \
+ && apt-get -yqq update \
+ && apt-get -yqq install --no-install-recommends logstash=$LOGSTASH_VERSION \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV PATH /opt/logstash/bin:$PATH
+
+# Get GeoIP DB
+#RUN set -x \
+# && wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz \
+# && gunzip GeoIP.dat.gz \
+# && mkdir -p /usr/share/GeoIP \
+# && mv GeoIP.dat /usr/share/GeoIP
+
+ADD logstash.conf /etc/logstash/conf.d/logstash.conf
+ADD userale.json /etc/logstash/conf.d/userale.json
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/logstash/logstash.conf
----------------------------------------------------------------------
diff --git a/docker/logstash/logstash.conf b/docker/logstash/logstash.conf
new file mode 100644
index 0000000..656245e
--- /dev/null
+++ b/docker/logstash/logstash.conf
@@ -0,0 +1,78 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+input {
+ http {
+ codec => "json"
+ port => 8888
+ response_headers => {
+ "Access-Control-Allow-Origin" => "*"
+ "Access-Control-Allow-Headers" => "Origin, X-Requested-With, Content-Type, Accept"
+ }
+ }
+
+ file {
+ path => [
+ "/var/log/sennsoft/*.log"
+ ]
+ codec => "json"
+ start_position => "beginning"
+ sincedb_path => "/dev/null"
+ }
+}
+
+## Add your filters / logstash plugins configuration here
+filter {
+ mutate {
+ remove_field => [ "message", "@version", "@timestamp"]
+ }
+
+ grok {
+ match => [ "useraleVersion", "(?<major_ver>\d+).(?<minor_ver>\d+)(.(?<patch_ver>\d+))?" ]
+ match => [ "toolVersion", "(?<tool_major_ver>\d+).(?<tool_minor_ver>\d+)(.(?<tool_patch_ver>\d+))?" ]
+ }
+
+ #dns {
+ # add_field => [ "hostname", "%{host}" ]
+ #}
+
+ #dns {
+ # resolve => [ "host" ]
+ # action => [ "replace" ]
+ #}
+
+ #geoip {
+ # source => "host"
+ # database => "/usr/share/GeoIP/GeoIP.dat"
+ # target => "geoip"
+ #}
+}
+
+output {
+ # Output data to Elasticsearch instance
+ elasticsearch {
+ hosts => "elasticsearch:9200"
+ action => "index"
+ index => "userale"
+ document_type => "logs"
+ manage_template => true
+ template_overwrite => true
+ template => "/etc/logstash/conf.d/userale.json"
+ template_name => "userale"
+ }
+
+ # Debug
+ stdout { codec => rubydebug }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/logstash/userale.json
----------------------------------------------------------------------
diff --git a/docker/logstash/userale.json b/docker/logstash/userale.json
new file mode 100644
index 0000000..3d53d7b
--- /dev/null
+++ b/docker/logstash/userale.json
@@ -0,0 +1,152 @@
+{
+ "template" : "userale",
+ "order" : 1,
+ "settings" : {
+ "number_of_shards" : 1,
+ "number_of_replicas" : 0,
+ "index.refresh_interval" : "5s"
+ },
+ "mappings" : {
+ "_default_" : {
+ "_all" : {
+ "enabled" : false
+ },
+ "properties" : {
+ "geoip" : {
+ "properties" : {
+ "ip" : {
+ "type" : "ip"
+ },
+ "country_code2" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "country_code3" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "country_name" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "continent_code" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "region_name" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "city_name" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "postal_code" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "latitude" : {
+ "type" : "double"
+ },
+ "longitude" : {
+ "type" : "double"
+ },
+ "dma_code" : {
+ "type" : "integer"
+ },
+ "area_code" : {
+ "type" : "long"
+ },
+ "timezone" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "real_region_name" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "location" : {
+ "doc_values" : true,
+ "type" : "geo_point"
+ }
+ }
+ }
+ }
+ },
+ "logs" : {
+ "properties" : {
+ "userAction" : {
+ "type" : "boolean"
+ },
+ "type" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "clientTime" : {
+ "type" : "date",
+ "format" : "strict_date_optional_time||epoch_millis"
+ },
+ "target" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "path" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "details" : {
+ "type" : "object"
+ },
+ "location" : {
+ "type" : "object",
+ "properties" : {
+ "x" : {
+ "type" : "integer"
+ },
+ "y" : {
+ "type" : "integer"
+ }
+ }
+ },
+ "userId" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "session" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "toolName" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "toolVersion" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "useraleVersion" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "major_ver" : {
+ "type" : "integer"
+ },
+ "minor_ver" : {
+ "type" : "integer"
+ },
+ "patch_ver" : {
+ "type" : "integer"
+ },
+ "tool_major_ver" : {
+ "type" : "integer"
+ },
+ "tool_minor_ver" : {
+ "type" : "integer"
+ },
+ "tool_patch_ver" : {
+ "type" : "integer"
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-tap/blob/1ca22406/docker/tap/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/tap/Dockerfile b/docker/tap/Dockerfile
index d6a06e6..7d5214a 100644
--- a/docker/tap/Dockerfile
+++ b/docker/tap/Dockerfile
@@ -58,5 +58,8 @@ ADD neon_graph.js /usr/src/app/public
# Export port
EXPOSE 8000
+# Migrate for django
+#RUN python manage.py migrate
+
# Startup Application
RUN gulp docker
\ No newline at end of file