You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@senssoft.apache.org by le...@apache.org on 2018/03/12 15:58:18 UTC
[04/40] incubator-senssoft git commit: Initial (probably broken)
commit of ported docker build
Initial (probably broken) commit of ported docker build
Project: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/commit/1c10c783
Tree: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/tree/1c10c783
Diff: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/diff/1c10c783
Branch: refs/heads/asf-site
Commit: 1c10c7835193e5f708a7720bf10e2e85e2a04c19
Parents: da63669
Author: Foley <rg...@draper.com>
Authored: Wed Mar 22 09:26:17 2017 -0400
Committer: Foley <rg...@draper.com>
Committed: Wed Mar 22 09:26:17 2017 -0400
----------------------------------------------------------------------
docker/db/Dockerfile | 21 ++
docker/db/sql.sh | 32 +++
docker/distill/Dockerfile | 50 ++++
docker/distill/distill.conf | 29 +++
docker/distill/distill/__init__.py | 45 ++++
docker/distill/distill/algorithms/__init__.py | 14 ++
.../distill/algorithms/graphs/__init__.py | 14 ++
.../distill/distill/algorithms/graphs/graph.py | 24 ++
.../distill/algorithms/graphs/tests/__init__.py | 22 ++
.../distill/algorithms/stats/__init__.py | 14 ++
docker/distill/distill/algorithms/stats/hist.py | 183 +++++++++++++++
.../distill/algorithms/stats/tests/__init__.py | 22 ++
.../distill/algorithms/tests/__init__.py | 14 ++
docker/distill/distill/app.py | 216 +++++++++++++++++
docker/distill/distill/config.cfg | 57 +++++
docker/distill/distill/models/__init__.py | 14 ++
docker/distill/distill/models/brew.py | 235 +++++++++++++++++++
docker/distill/distill/models/stout.py | 149 ++++++++++++
docker/distill/distill/models/tests/__init__.py | 22 ++
docker/distill/distill/models/userale.py | 137 +++++++++++
docker/distill/distill/server.py | 29 +++
docker/distill/distill/tests/__init__.py | 21 ++
docker/distill/distill/tests/basic_test.py | 24 ++
docker/distill/distill/tests/distill_test.py | 43 ++++
docker/distill/distill/utils/__init__.py | 0
docker/distill/distill/utils/exceptions.py | 25 ++
docker/distill/distill/utils/query_builder.py | 35 +++
docker/distill/distill/utils/tests/__init__.py | 21 ++
docker/distill/distill/utils/validation.py | 39 +++
docker/distill/distill/version.py | 22 ++
docker/distill/requirements.txt | 22 ++
docker/distill/setup.cfg | 30 +++
docker/distill/setup.py | 87 +++++++
docker/docker-compose.yml | 78 ++++++
docker/es/._elasticsearch.yml | Bin 0 -> 222 bytes
docker/es/Dockerfile | 5 +
docker/es/elasticsearch.yml | 98 ++++++++
docker/kibana/Dockerfile | 7 +
docker/kibana/entrypoint.sh | 10 +
docker/logstash/Dockerfile | 11 +
docker/logstash/config/logstash-apache.conf | 82 +++++++
docker/logstash/config/logstash-userale.conf | 55 +++++
docker/logstash/templates/apache.json | 59 +++++
docker/logstash/templates/userale.json | 157 +++++++++++++
docker/tap/Dockerfile | 63 +++++
docker/tap/README.md | 2 +
docker/tap/neon_counts.js | 2 +
docker/tap/neon_graph.js | 2 +
48 files changed, 2343 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/db/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/db/Dockerfile b/docker/db/Dockerfile
new file mode 100644
index 0000000..bb6152f
--- /dev/null
+++ b/docker/db/Dockerfile
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM postgres:latest
+MAINTAINER Michelle Beard <ms...@apache.org>
+
+ADD sql.sh docker-entrypoint-initdb.d/sql.sh
+
+EXPOSE 5432
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/db/sql.sh
----------------------------------------------------------------------
diff --git a/docker/db/sql.sh b/docker/db/sql.sh
new file mode 100644
index 0000000..4de31b5
--- /dev/null
+++ b/docker/db/sql.sh
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+echo "Creating POSTGRES DB FROM ENVIRONMENT"
+DB_NAME=tapdb
+DB_USER=tapuser
+DB_PASS="Dr@p3rUs3r"
+DB_SERVICE=postgres
+
+psql -U postgres -c "CREATE USER $DB_USER PASSWORD '$DB_PASS'"
+psql -U postgres -c "CREATE DATABASE $DB_NAME OWNER $DB_USER"
+psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE $DB_NAME TO $DB_USER"
+
+#localedef -i de_DE -c -f UTF-8 -A /usr/share/locale/locale.alias de_DE.UTF-8
+##export LANG=en_US.UTF-8
+#locale # confirm that it shows only en_US.UTF-8 for all settings
+# finally, run your opennms installer
+#/usr/share/opennms/bin/install -l /usr/local/lib -dis
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/distill/Dockerfile b/docker/distill/Dockerfile
new file mode 100644
index 0000000..f69edc8
--- /dev/null
+++ b/docker/distill/Dockerfile
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:2
+
+# install system wide deps
+RUN apt-get -yqq update
+
+# Set the work directory
+RUN mkdir -p /usr/src
+WORKDIR /usr/src
+
+# Install git
+##RUN sudo -E apt-get -yqq install \
+ ##git
+
+# Clone Distill
+RUN git clone https://github.com/apache/incubator-senssoft-distill.git distill
+WORKDIR /usr/src/distill
+RUN git pull
+
+
+# Add code
+##WORKDIR /app
+##ADD setup.py /app
+
+ADD requirements.txt /distill
+
+# Install Distill dependencies
+RUN pip install -r requirements.txt
+
+# Expose Ports
+EXPOSE 8090
+
+# Rest
+ADD . /app
+RUN python setup.py develop
+CMD python distill/server.py
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill.conf
----------------------------------------------------------------------
diff --git a/docker/distill/distill.conf b/docker/distill/distill.conf
new file mode 100644
index 0000000..a113dcb
--- /dev/null
+++ b/docker/distill/distill.conf
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+description "Gunicorn application server running Apache Distill"
+
+# Restart process if it ever fails
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+#Setup user and group that Gunicorn should be run as
+setuid nobody
+setgid www-data
+
+# Path to run_server
+chdir /path/to/distill
+exec gunicorn -c "gunicorn.cfg" scripts/run_server:app
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/__init__.py b/docker/distill/distill/__init__.py
new file mode 100644
index 0000000..2b44372
--- /dev/null
+++ b/docker/distill/distill/__init__.py
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from flask import Flask
+from elasticsearch_dsl.connections import connections
+
+# Initialize Flask instance
+app = Flask (__name__)
+
+# Load Configurations
+app.config.from_pyfile('config.cfg')
+
+# Unpack Elasticsearch configuration and create elasticsearch connection
+host = app.config ['ES_HOST']
+port = app.config ['ES_PORT']
+http_auth = app.config ['HTTP_AUTH']
+use_ssl = app.config ['USE_SSL']
+verify_certs = app.config ['VERIFY_CERTS']
+ca_certs = app.config ['CA_CERTS']
+client_cert = app.config ['CLIENT_CERT']
+client_key = app.config ['CLIENT_KEY']
+timeout = app.config ['TIMEOUT']
+
+# Initialize Elasticsearch instance
+es = connections.create_connection (hosts = [host],
+ port = port,
+ http_auth = http_auth,
+ use_ssl = use_ssl,
+ verify_certs = verify_certs,
+ ca_certs = ca_certs,
+ client_cert = client_cert,
+ client_key = client_key,
+ timeout=timeout)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/algorithms/__init__.py b/docker/distill/distill/algorithms/__init__.py
new file mode 100644
index 0000000..6acb5d1
--- /dev/null
+++ b/docker/distill/distill/algorithms/__init__.py
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/graphs/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/algorithms/graphs/__init__.py b/docker/distill/distill/algorithms/graphs/__init__.py
new file mode 100644
index 0000000..6acb5d1
--- /dev/null
+++ b/docker/distill/distill/algorithms/graphs/__init__.py
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/graphs/graph.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/algorithms/graphs/graph.py b/docker/distill/distill/algorithms/graphs/graph.py
new file mode 100644
index 0000000..3c44730
--- /dev/null
+++ b/docker/distill/distill/algorithms/graphs/graph.py
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class GraphAnalytics (object):
+ """
+ Distill's graph analytics package. Apply graph algorithms to User Ale log data segmented with
+ Stout.
+ """
+
+ @staticmethod
+ def foo ():
+ pass
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/graphs/tests/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/algorithms/graphs/tests/__init__.py b/docker/distill/distill/algorithms/graphs/tests/__init__.py
new file mode 100644
index 0000000..f6f6899
--- /dev/null
+++ b/docker/distill/distill/algorithms/graphs/tests/__init__.py
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+distill: tests module.
+
+Meant for use with py.test.
+Organize tests into files, each named xxx_test.py
+Read more here: http://pytest.org/
+'''
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/stats/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/algorithms/stats/__init__.py b/docker/distill/distill/algorithms/stats/__init__.py
new file mode 100644
index 0000000..6acb5d1
--- /dev/null
+++ b/docker/distill/distill/algorithms/stats/__init__.py
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/stats/hist.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/algorithms/stats/hist.py b/docker/distill/distill/algorithms/stats/hist.py
new file mode 100644
index 0000000..b516423
--- /dev/null
+++ b/docker/distill/distill/algorithms/stats/hist.py
@@ -0,0 +1,183 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from distill import es
+from distill.utils.query_builder import QueryBuilder
+from flask import jsonify
+from elasticsearch import Elasticsearch, TransportError
+
+class Hist (object):
+ """
+ Distill's statistics package. Apply statistical algorithms to User Ale log data segmented with
+ Stout. Need to query/filter by session or user id.
+ """
+
+ def __init__ (self):
+ # parse out query
+ pass
+
+ # @staticmethod
+ # def filter (app, app_type=None, q=''):
+
+ # field = q.get ("field") if q.get ("field") else ""
+ # size = q.get ("size") if q.get ("size") else 10
+
+ # query = { "aggs" : {
+ # "count_by_type" : {
+ # "filter" : { "term" : { field : }}
+ # "terms" : {
+ # "field" : field,
+ # "size" : 100
+ # }
+ # }
+ # }
+ # }
+
+ # d = {}
+ # # try:
+ # response = es.search (index=app, doc_type=app_type, body=query)
+ # # for tag in response['aggregations']['count_by_type']['buckets']:
+ # # d [tag ['key']] = tag ['doc_count']
+ # # except TransportError as e:
+ # # d ['error'] = e.info
+ # # except Exception as e:
+ # # d ['error'] = str (e)
+ # # return jsonify (d)
+ # return jsonify (response)
+
+ @staticmethod
+ def terms (app, app_type=None, q=''):
+ """
+ Group by field (find all elements )
+ """
+ field = q.get ("field") if q.get ("field") else ""
+ segment = q.get ("seg") if q.get ("seg") else "*"
+ size = q.get ("size") if q.get ("size") else 10000
+ numhits = q.get ("numhits") if q.get ("numhits") else 10
+
+ query = { "aggs" : {
+ "count_by_type" : {
+ "terms" : {
+ "field" : field,
+ "size" : size # maximum number of keys (unique fields)
+ },
+ "aggs" : {
+ "top" : { # arbitrary name
+ "top_hits" : {
+ "size" : numhits, # number of logs in subgroup
+ "_source" : { # segment on fields - return only subgroup based on field
+ "include" : [
+ segment
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ d = {}
+ # try:
+ response = es.search (index=app, doc_type=app_type, body=query)
+ # for tag in response['aggregations']['count_by_type']['buckets']:
+ # d [tag ['key']] = tag ['doc_count']
+ # except TransportError as e:
+ # d ['error'] = e.info
+ # except Exception as e:
+ # d ['error'] = str (e)
+ # return jsonify (d)
+ return jsonify (response)
+
+ @staticmethod
+ def unique_terms (app, app_type=None, q=""):
+ """
+ Aggregate the number of unique terms in a field. Missing values are counted and marked as "N/A".
+
+ .. todo::
+
+ Need to incorporate QueryBuilder library instead of manually generating queries.
+
+ :param app: [string] application name
+ :param app_type: [string] application type
+ :param field: [string] field to search against for unique values
+ :param size: [int] the top size terms returned in the result. Default value is 10.
+ :param min_hits: [int] return tags which have been found in min_hits or more. Default value is 1.
+ :return: [dict] dictionary of results
+ """
+
+ field = q.get ("field") if q.get ("field") else ""
+ size = q.get ("size") if q.get ("size") else 10000
+ min_hits = q.get ("min_hits") if q.get ("min_hits") else 0
+
+ print field
+ query = { "aggs" : {
+ "terms_agg" : {
+ "terms" : {
+ "field" : field,
+ "size" : size,
+ "min_doc_count" : min_hits,
+ "missing" : "N/A"
+ }
+ }
+ }
+ }
+
+ d = {}
+ try:
+ response = es.search (index=app, doc_type=app_type, body=query)
+ for tag in response['aggregations']['terms_agg']['buckets']:
+ d [tag ['key']] = tag ['doc_count']
+ except TransportError as e:
+ d ['error'] = e.info
+ except Exception as e:
+ d ['error'] = str (e)
+ return jsonify (d)
+
+ @staticmethod
+ def histogram (app, app_type=None, q=""):
+ """
+ Only works on numerical data.
+ """
+ field = q.get ("field") if q.get ("field") else ""
+
+ interval = 50
+ query = { "aggs" : {
+ "hist_agg" : {
+ "histogram" : {
+ "field" : field,
+ "interval" : interval
+ }
+ }
+ }
+ }
+
+ d = {}
+ try:
+ response = es.search (index=app, doc_type=app_type, body=query)
+ for tag in response['aggregations']['hist_agg']['buckets']:
+ d [tag ['key']] = tag ['doc_count']
+ except TransportError as e:
+ d ['error'] = e.info
+ except Exception as e:
+ d ['error'] = str (e)
+ return jsonify (d)
+
+ def get_value ():
+ return 0
+
+ def _parse_msg (query):
+ # should have form ?measure=name&field=f1, f2&event=a,b
+ pass
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/stats/tests/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/algorithms/stats/tests/__init__.py b/docker/distill/distill/algorithms/stats/tests/__init__.py
new file mode 100644
index 0000000..f6f6899
--- /dev/null
+++ b/docker/distill/distill/algorithms/stats/tests/__init__.py
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+distill: tests module.
+
+Meant for use with py.test.
+Organize tests into files, each named xxx_test.py
+Read more here: http://pytest.org/
+'''
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/tests/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/algorithms/tests/__init__.py b/docker/distill/distill/algorithms/tests/__init__.py
new file mode 100644
index 0000000..6acb5d1
--- /dev/null
+++ b/docker/distill/distill/algorithms/tests/__init__.py
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/app.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/app.py b/docker/distill/distill/app.py
new file mode 100644
index 0000000..58434a1
--- /dev/null
+++ b/docker/distill/distill/app.py
@@ -0,0 +1,216 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from flask import Flask, request, jsonify
+from distill import app
+from distill.models.brew import Brew
+from distill.models.userale import UserAle
+from distill.models.stout import Stout
+from distill.algorithms.stats.hist import Hist
+
+@app.route ('/', methods=['GET'])
+def index ():
+ """
+ Show Distill version information, connection status, and all registered applications.
+
+ .. code-block:: bash
+
+ $ curl -XGET https://localhost:8090
+
+ {
+ "author" : "Michelle Beard",
+ "email" : "mbeard@draper.com",
+ "name": "Distill",
+ "status" : true,
+ "version" : "1.0",
+ "applications" : {
+ "xdata_v3" : {
+ testing: 205,
+ parsed: 500,
+ },
+ "test_app" : {
+ logs: 500,
+ parsed: 100,
+ }
+ }
+ }
+
+ :return: Distill's status information as JSON blob
+ """
+ return jsonify (name="Distill", version="1.0 alpha", author="Michelle Beard", email="mbeard@draper.com", status=Brew.get_status (), applications=Brew.get_applications ())
+
+@app.route ('/create/<app_id>', methods=['POST', 'PUT'])
+def create (app_id):
+ """
+ Registers an application in Distill.
+
+ .. code-block:: bash
+
+ $ curl -XPOST https://localhost:8090/xdata_v3
+
+ :param app_id: Application name
+ :return: Newly created application's status as JSON blob
+ """
+ return Brew.create (app_id)
+
+@app.route ('/status/<app_id>', defaults={"app_type" : None}, methods=['GET'])
+@app.route ('/status/<app_id>/<app_type>', methods=['GET'])
+def status (app_id, app_type):
+ """
+ Presents meta information about an registered application, including field names and document types.
+
+ .. code-block:: bash
+
+ $ curl -XGET https://localhost:8090/status/xdata_v3
+
+ {
+ "application": "xdata_v3",
+ "health": "green",
+ "num_docs": "433",
+ "status": "open"
+ }
+
+ :param app_id: Application name
+ :return: Registered applications meta data as JSON blob
+ """
+ return Brew.read (app_id, app_type=app_type)
+
+@app.route ('/update/<app_id>', methods=['POST', 'PUT'])
+def update (app_id):
+ """
+ Renames a specific application
+
+ .. code-block:: bash
+
+ $ curl -XPOST https://localhost:8090/update/xdata_v3?name="xdata_v4"
+
+ :param app_id: Application name
+ :return: Boolean response message as JSON blob
+ """
+ return Brew.update (app_id)
+
+@app.route ('/delete/<app_id>', methods=['DELETE'])
+def delete (app_id):
+ """
+ Deletes an application permentantly from Distill
+
+ .. code-block:: bash
+
+ $ curl -XDELETE https://localhost:8090/xdata_v3
+
+ :param app_id: Application name
+ :return: Boolean response message as JSON blob
+ """
+ return Brew.delete (app_id)
+
+@app.route ('/search/<app_id>', defaults={"app_type" : None}, methods=['GET'])
+@app.route ('/search/<app_id>/<app_type>', methods=['GET'])
+def segment (app_id, app_type):
+ """
+ Search against an application on various fields.
+
+ .. code-block:: bash
+
+ $ curl -XGET https://[hostname]:[port]/search/xdata_v3?q=session_id:A1234&size=100&scroll=false&fl=param1,param2
+
+ :param app_id: Application name
+ :param app_type: Optional document type to filter against
+ :param q: Main search query. To return all documents, pass in q=*:*
+ :param size: Maximum number of documents to return in request
+ :param scroll: Scroll id if the number of documents exceeds 10,000
+ :param fl: List of fields to restrict the result set
+ :return: JSON blob of result set
+ """
+ q = request.args
+ return UserAle.segment (app_id, app_type=app_type, params=q)
+
+@app.route ('/stat/<app_id>', defaults={"app_type" : None}, methods=['GET'])
+@app.route ('/stat/<app_id>/<app_type>', methods=['GET'])
+def stat (app_id, app_type):
+ """
+ Generic histogram counts for a single registered application filtered optionally by document type.
+ View the Statistics document page for method definitions and arguments
+
+ .. code-block:: bash
+
+ $ curl -XGET https://localhost:8090/stat/xdata_v3/testing/?stat=terms&elem=signup&event=click
+
+ :param app_id: Application name
+ :param app_type: Application type
+ :return: JSON blob of result set
+ """
+ stat = request.args.get ('stat')
+ q = request.args
+
+ hist_cls = Hist ()
+ method = None
+ try:
+ method = getattr (hist_cls, stat)
+ return method (app_id, app_type, q=q)
+ except AttributeError:
+ msg = "Class `{}` does not implement `{}`".format(hist_cls.__class__.__name__, stat)
+ return jsonify (error=msg)
+
+@app.route ('/denoise/<app_id>', methods=['GET'])
+def denoise (app_id):
+ """
+ Bootstrap script to cleanup the raw logs. A document type called "parsed"
+ will be stored with new log created unless specified in the request. Have option to save
+ parsed results back to data store. These parsed logs can be intergrated with STOUT results
+ by running the stout bootstrap script.
+
+ .. code-block:: bash
+
+ $ curl -XGET https://localhost:8090/denoise/xdata_v3?save=true&type=parsed
+
+ :param app_id: Application name
+ :return: [dict]
+ """
+ doc_type = 'parsed'
+ save = False
+ q = request.args
+ if 'save' in q:
+ save = str2bool (q.get ('save'))
+ if 'type' in q:
+ # @TODO: Proper cleanup script needs to happen
+ doc_type = q.get ('type')
+ return UserAle.denoise (app_id, doc_type=doc_type, save=save)
+
+@app.route ('/stout', methods=['GET'])
+def merge_stout ():
+ """
+ Bootstrap script to aggregate user ale logs to stout master answer table
+ This will save the merged results back to ES instance at new index stout
+ OR denoise data first, then merge with the stout index...
+ If STOUT is enabled, the select method expects a stout index to exist or otherwise
+ it will return an error message.
+
+ .. code-block:: bash
+
+ $ curl -XGET https://locahost:8090/stout/xdata_v3
+
+ :return: Status message
+ """
+ flag = app.config ['ENABLE_STOUT']
+ if flag:
+ return Stout.ingest ()
+ return jsonify (status="STOUT is disabled.")
+
+@app.errorhandler(404)
+def page_not_found (error):
+ """
+ Generic Error Message
+ """
+ return "Unable to find Distill."
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/config.cfg
----------------------------------------------------------------------
diff --git a/docker/distill/distill/config.cfg b/docker/distill/distill/config.cfg
new file mode 100644
index 0000000..189e3ea
--- /dev/null
+++ b/docker/distill/distill/config.cfg
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Statement for enabling the development environment
+DEBUG = True
+
+# Host
+HOST = '0.0.0.0'
+
+# Port
+PORT = 8090
+
+# Enable STOUT integration into Distill
+ENABLE_STOUT = False
+SQLITEDB = '../path/to/stout/stout.db'
+MASTER = '../path/to/master/master_ans.csv'
+MAPPINGS = '../path/to/mappings/MOT_Mappings.csv'
+SELECTED = '../path/to/stout/selected_vars_for_distill.csv'
+
+# Elasticsearch Configuration
+ES_HOST = 'http://elasticsearch'
+ES_PORT = 9200
+HTTP_AUTH = None
+USE_SSL = False
+VERIFY_CERTS = False
+CA_CERTS = None
+CLIENT_CERT = None
+CLIENT_KEY = None
+TIMEOUT = 3
+
+# Application threads. A common general assumption is
+# using 2 per available processor cores - to handle
+# incoming requests using one and performing background
+# operations using the other.
+THREADS_PER_PAGE = 2
+
+# Enable protection agains *Cross-site Request Forgery (CSRF)*
+# CSRF_ENABLED = True
+
+# Use a secure, unique and absolutely secret key for
+# signing the data.
+# CSRF_SESSION_KEY = "secret"
+
+# Secret key for signing cookies
+# SECRET_KEY = "secret"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/models/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/models/__init__.py b/docker/distill/distill/models/__init__.py
new file mode 100644
index 0000000..6acb5d1
--- /dev/null
+++ b/docker/distill/distill/models/__init__.py
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/models/brew.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/models/brew.py b/docker/distill/distill/models/brew.py
new file mode 100644
index 0000000..28d16b3
--- /dev/null
+++ b/docker/distill/distill/models/brew.py
@@ -0,0 +1,235 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from elasticsearch import Elasticsearch, TransportError
+from flask import jsonify
+from distill import es
+
+class Brew (object):
+ """
+ Distill supports basic CRUD operations and publishes the status
+ of an persistenct database. Eventually it will support ingesting logs sent from
+ an registered application.
+ """
+
+ @staticmethod
+ def get_status ():
+ """
+ Fetch the status of the underlying database instance.
+
+ :return: [bool] if connection to database instance has been established
+ """
+ return es.ping (ignore=[400, 404])
+
+ @staticmethod
+ def get_applications ():
+ """
+ Fetch all the registered applications in Distill.
+
+ .. note:: Private indexes starting with a period are not included in the result set
+
+ :return: [dict] dictionary of all registered applications and meta information
+ """
+ doc = {}
+ query = { "aggs" : {
+ "count_by_type" : {
+ "terms" : {
+ "field" : "_type",
+ "size" : 100
+ }
+ }
+ }
+ }
+
+ try:
+ cluster_status = es.cat.indices (h=["index"], pri=False)
+ x = cluster_status.splitlines()
+
+ for idx in x:
+ idx = idx.rstrip ()
+
+ # Ignore private indexes (like .kibana or .stout)
+ if idx [:1] != '.':
+ response = es.search (index=idx, body=query)
+ d = {}
+ for tag in response["aggregations"]["count_by_type"]["buckets"]:
+ d [tag ['key']] = tag ['doc_count']
+ doc [idx] = d
+ except TransportError as e:
+ doc ['error'] = e.info
+ except Exception as e:
+ doc ['error'] = str (e)
+ return doc
+
+ @staticmethod
+ def create (app):
+ """
+ Register a new application in Distill
+
+ .. code-block:: bash
+
+ {
+ "application" : "xdata_v3",
+ "health" : "green",
+ "num_docs" : 0,
+ "status" : "open"
+ }
+
+ :param app: [string] application name (e.g. xdata_v3)
+ :return: [dict] dictionary of application and its meta information
+ """
+
+ # ignore 400 cause by IndexAlreadyExistsException when creating an index
+ res = es.indices.create (index=app, ignore=[400, 404])
+ doc = _get_cluster_status (app)
+ return jsonify (doc)
+
+ @staticmethod
+ def read (app, app_type=None):
+ """
+ Fetch meta data associated with an application
+
+ .. code-block:: bash
+
+ Example:
+ {
+ "application" : "xdata_v3",
+ "health" : "green",
+ "num_docs" : "100",
+ "status" : "open"
+ "types" : {
+ "raw_logs" : {
+ "@timestamp" : "date",
+ "action" : "string",
+ "elementId" : "string"
+ },
+ "parsed" : {
+ "@timestamp" : "date",
+ "elementId_interval" : "string"
+ },
+ "graph" : {
+ "uniqueID" : "string",
+ "transition_count" : "long",
+ "p_value" : "float"
+ }
+ }
+ }
+
+ :param app: [string] application name (e.g. xdata_v3)
+ :return: [dict] dictionary of application and its meta information
+ """
+
+ return jsonify (_get_cluster_status (app, app_type=app_type))
+
+ @staticmethod
+ def update (app):
+ """
+ .. todo::
+ Currently not implemented
+ """
+
+ return jsonify (status="not implemented")
+
+ @staticmethod
+ def delete (app):
+ """
+ Technically closes the index so its content is not searchable.
+
+ .. code-block: bash
+
+ Example:
+ {
+ status: "Deleted index xdata_v3"
+ }
+
+ :param app: [string] application name (e.g. xdata_v3)
+ :return: [dict] status message of the event
+ """
+
+ es.indices.close (index=app, ignore=[400, 404])
+ return jsonify (status="Deleted index %s" % app)
+
+def _get_cluster_status (app, app_type=None):
+ """
+ Return cluster status, index health, and document count as string
+
+ @todo figure out how to count individual documents stored at an app_type (currently shows only index count)
+ :param app: [string] application name (e.g. xdata_v3)
+ :return: [dict] dictionary of index meta data including field names
+ """
+
+ doc = {}
+ try:
+ cluster_status = es.cat.indices (index=app, h=["health", "status", "docs.count"], pri=True, ignore=[400, 404])
+ v = str (cluster_status).split (" ")
+ m = ["health", "status", "num_docs"]
+ doc = dict (zip (m, v))
+ # Add back application
+ doc ["application"] = app
+ except TransportError as e:
+ doc ['error'] = e.info
+ except Exception as e:
+ doc ['error'] = str (e)
+
+ doc ['fields'] = _get_all_fields (app, app_type)
+ return doc
+
+def _parse_mappings (app, app_type=None):
+ """
+ .. todo:
+
+ Need to parse out result set that presents field list and type
+ """
+
+ try:
+ mappings = es.indices.get_mapping (index=app, doc_type=[app_type], ignore=[400, 404])
+ # mappings = yaml.safe_load (json.ess (mappings))
+ # print json.dumps (mappings [app]["mappings"], indent=4, separators=(',', ': '))
+ ignore = ["properties", "format"]
+ except TransportError as e:
+ doc ['error'] = e.info
+ except Exception as e:
+ doc ['error'] = str (e)
+ return doc
+
+def _get_all_fields (app, app_type=None):
+ """
+ Retrieve all possible fields in an application
+
+ :param app: [string] application name (e.g. xdata_v3)
+ :param app_type: [string] application type (e.g. logs)
+ :return: [list] list of strings representing the fields names
+ """
+ d = list ()
+ query = { "aggs" : {
+ "fields" : {
+ "terms" : {
+ "field" : "_field_names",
+ "size" : 100
+ }
+ }
+ }
+ }
+
+ try:
+ response = es.search (index=app, doc_type=app_type, body=query)
+ for tag in response['aggregations']['fields']['buckets']:
+ d.append (tag ['key'])
+ except TransportError as e:
+ d.append (str (e.info))
+ except Exception as e:
+ d.append (str (e))
+ return d
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/models/stout.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/models/stout.py b/docker/distill/distill/models/stout.py
new file mode 100644
index 0000000..d6421d8
--- /dev/null
+++ b/docker/distill/distill/models/stout.py
@@ -0,0 +1,149 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from distill import app, es
+from elasticsearch_dsl import DocType, String, Boolean, Date, Nested, Search
+from elasticsearch_dsl.query import MultiMatch, Match, Q
+from elasticsearch import Elasticsearch, TransportError
+from flask import jsonify
+import pandas as pd
+
+class StoutDoc (DocType):
+ """
+ Representation of a Stout documentat.
+ """
+
+ sessionID = String (index="not_analyzed")
+ task1 = Nested ()
+ task2 = Nested ()
+
+ class Meta:
+ index = '.stout'
+ doc_type = 'testing'
+
+ def save (self, *args, **kwargs):
+ """
+ Save data from parsing as a Stout document in Distill
+ """
+ return super (StoutDoc, self).save (*args, **kwargs)
+
+class Stout (object):
+ """
+ Main Stout class to support ingest and search operations.
+ """
+
+ @staticmethod
+ def ingest ():
+ """
+ Ingest data coming from Stout to Distill
+ """
+
+ # Create the mappings in elasticsearch
+ StoutDoc.init ()
+ status = True
+ data = _parse ();
+ try:
+ for k,v in data.items ():
+ doc = StoutDoc ()
+ if 'sessionID' in v:
+ doc.sessionID = v['sessionID']
+ if 'task1' in v:
+ doc.task1 = v['task1']
+ if 'task2' in v:
+ doc.task2 = v['task2']
+ doc.save ()
+ except Error as e:
+ status = False
+ return jsonify (status=status)
+
+def _parse ():
+ """
+ Parse master answer table with mapping into an associative array
+
+ :return: [dict] dictionary of session information
+ """
+ master = app.config ['MASTER']
+ mappings = app.config ['MAPPINGS']
+
+ fileContents=pd.read_csv(master, encoding='utf-8')
+ plainTextMappings=pd.read_csv(mappings, encoding='raw_unicode_escape')
+ headers=list(fileContents.columns.values)
+
+ #generate the mapping between header and plain text
+ translationRow={};
+ for fieldIndex in range(1,len(headers)):
+ t=plainTextMappings.ix[fieldIndex]
+ translationRow[headers[fieldIndex]]=t[9]
+
+ dictBySessionID={}
+ translationRow['items.text']='foo'
+ index=0
+ for row in fileContents.iterrows():
+ index=index+1
+
+ taskMetrics={}
+ index,data=row
+ identifier=row[1][0].split("::")
+ sessionID=identifier[0]
+ taskID=(identifier[1])
+ workingData={}
+ #is this session id already in the dictionary?
+ if sessionID in dictBySessionID:
+ #grab the entry as workingData
+ workingData=dictBySessionID[sessionID]
+
+ sysData={}
+ task1Data={}
+ task2Data={}
+ metaData={}
+ d={}
+
+ for fieldIndex in range(1,len(headers)):
+ if not pd.isnull(row[1][fieldIndex]): #only interested in non-null fields
+ tempDict={}
+ if headers[fieldIndex] in translationRow:
+ tempDict['field']=translationRow[headers[fieldIndex]]
+ #tempDict['field']=translationRow[9]
+ tempDict['value']=row[1][fieldIndex]
+ d[headers[fieldIndex]]=row[1][fieldIndex]
+ if "SYS" in headers[fieldIndex]:
+ sysData[headers[fieldIndex]]=tempDict
+ elif "OT1" in headers[fieldIndex]:
+ task1Data[headers[fieldIndex]]=tempDict
+ elif "OT2" in headers[fieldIndex]:
+ task2Data[headers[fieldIndex]]=tempDict
+ else:
+ metaData[headers[fieldIndex]]=tempDict
+
+ if d['TSK_TIME_DIFF_']>0: #block tasks with zero time elapsed
+ a=int(d['TSK_TIME_DIFF_OT1_'])
+ b=int(d['TSK_TIME_DIFF_OT2_'])
+ #figure out which task the values belong to
+ if ((a>0) & (b<=0)):
+ task1Data['taskID']=taskID
+ task1Data['meta']=metaData
+ task1Data['system']=sysData
+ workingData['task1']=task1Data
+ elif ((a<=0) & (b>0)):
+ task2Data['taskID']=taskID
+ task2Data['meta']=metaData
+ task2Data['system']=sysData
+ workingData['task2']=task2Data
+ else:
+ raise ValueError('Encountered an unexpected task time diff state')
+
+ workingData['sessionID'] = sessionID
+ dictBySessionID[sessionID]=workingData
+ return dictBySessionID
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/models/tests/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/models/tests/__init__.py b/docker/distill/distill/models/tests/__init__.py
new file mode 100644
index 0000000..f6f6899
--- /dev/null
+++ b/docker/distill/distill/models/tests/__init__.py
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+distill: tests module.
+
+Meant for use with py.test.
+Organize tests into files, each named xxx_test.py
+Read more here: http://pytest.org/
+'''
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/models/userale.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/models/userale.py b/docker/distill/distill/models/userale.py
new file mode 100644
index 0000000..f63fa51
--- /dev/null
+++ b/docker/distill/distill/models/userale.py
@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from elasticsearch import Elasticsearch, TransportError
+from elasticsearch_dsl import DocType, String, Boolean, Date, Float, Search
+from elasticsearch_dsl.query import MultiMatch, Match, Q
+from elasticsearch import Elasticsearch, TransportError
+from elasticsearch_dsl.connections import connections
+from werkzeug.datastructures import ImmutableMultiDict, MultiDict
+
+from flask import jsonify, Markup
+from distill import app, es
+import datetime
+
+class UserAle (object):
+ """
+ Main method of entry to perform segmentation and integration of STOUT's master
+ answer table (if STOUT is enabled). Advanced and basic analytics is performed in the
+ distill.algorithms.stats and distill.algorithms.graphs module.
+ """
+
+ @staticmethod
+ def segment (app, app_type=None, params=''):
+ """
+ Just support match all for now.
+ """
+ q = params.get ("q") if params.get ("q") else {}
+ fields = params.get ("fields") if params.get ("fields") else []
+ size = params.get ("size") if params.get ("size") else 10
+ scroll = params.get ("scroll") if params.get ("scroll") else False
+ fl = params.get ("fl") if params.get ("fl") else []
+
+ # filters = params.get ("filter") if params.get ("filter") else {}
+
+ # 'q': args.get('q', '{}'),
+ # 'fields': args.get('fl', '{}'),
+ # 'size': args.get ('size', 100),
+ # 'scroll': args.get ('scroll', False),
+ # 'filters': request_args.getlist ('fq')
+ query = {}
+ query ['size'] = size
+
+ if q:
+ res = q.split(":")
+ key = res [0]
+ val = res [1]
+ query ['query'] = {"match" : { key : val } }
+ else:
+ query ['query'] = {"match_all" : {}}
+
+ if len (fields) > 0:
+ ex = {
+ "include" : fields.split(",")
+ }
+ query ['_source'] = ex
+
+
+ response = es.search (index=app, doc_type=app_type, body=query)
+
+ return jsonify (response)
+
+ @staticmethod
+ def search (app,
+ app_type=None,
+ filters=list (),
+ size=100,
+ include="*",
+ scroll=None,
+ sort_field=None):
+ """
+ Perform a search query.
+
+ :param app: [string] application id (e.g. "xdata_v3")
+ :param app_type: [string] name of the application type. If None all application types are searched.
+ :param filters: [list of strings] list of filters for a query.
+ :param size: [int] maximum number of hits that should be returned
+ :param sort_field: [string] sorting field. Currently supported fields: "timestamp", "date"
+ :return: [dict] dictionary with processed results. If STOUT is enabled, STOUT data will be merged with final result.
+ """
+
+ # Need some query builder...
+ log_result = es.search (index=app, doc_type=app_type, body=query, fields=filters, size=size)
+
+ stout_result = Stout.getSessions ()
+
+ data = merged_results (log_result, stout_result)
+ return data
+
+ @staticmethod
+ def denoise (app, app_type='parsed', save=False):
+ """
+ """
+ pass
+
+"""
+Combine a list of dictionaries together to form one complete dictionary
+"""
+def merge_dicts (lst):
+ dall = {}
+ for d in lst:
+ dall.update (d)
+ return dall
+
+"""
+Get query parameters from the request and preprocess them.
+:param [dict-like structure] Any structure supporting get calls
+:result [dict] Parsed parameters
+"""
+def parse_query_parameters (indx, app_type=None, request_args = {}):
+ args = {key: value[0] for (key, value) in dict (request_args).iteritems ()}
+
+ # print "args = ", args
+ # Parse out simple filter queries
+ filters = []
+ for filter in get_all_fields (indx, app_type):
+ if filter in args:
+ filters.append((filter, args[filter]))
+
+ return {
+ 'q': args.get('q', '{}'),
+ 'fields': args.get('fl', []),
+ 'size': args.get ('size', 100),
+ 'scroll': args.get ('scroll', False),
+ 'filters': request_args.getlist ('fq')
+ }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/server.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/server.py b/docker/distill/distill/server.py
new file mode 100644
index 0000000..23acd83
--- /dev/null
+++ b/docker/distill/distill/server.py
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from distill import app
+from distill.app import *
+
+"""
+Start up a local WSGI server called development
+"""
+def dev_server ():
+ host = app.config ['HOST']
+ port = app.config ['PORT']
+ debug = app.config ['DEBUG']
+ app.run (host=host, port=port, debug=debug)
+
+if __name__ == '__main__':
+ dev_server ()
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/tests/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/tests/__init__.py b/docker/distill/distill/tests/__init__.py
new file mode 100644
index 0000000..09c5e2f
--- /dev/null
+++ b/docker/distill/distill/tests/__init__.py
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''
+distill: tests module.
+
+Meant for use with py.test.
+Organize tests into files, each named xxx_test.py
+Read more here: http://pytest.org/
+'''
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/tests/basic_test.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/tests/basic_test.py b/docker/distill/distill/tests/basic_test.py
new file mode 100644
index 0000000..712d1fe
--- /dev/null
+++ b/docker/distill/distill/tests/basic_test.py
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''
+distill: Test module.
+
+Meant for use with py.test.
+Write each test as a function named test_<something>.
+Read more here: http://pytest.org/
+'''
+
+def test_example():
+ assert True
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/tests/distill_test.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/tests/distill_test.py b/docker/distill/distill/tests/distill_test.py
new file mode 100644
index 0000000..2fb6502
--- /dev/null
+++ b/docker/distill/distill/tests/distill_test.py
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from flask import Flask, request
+
+from distill import app as test_app
+
+def test_example ():
+ assert True
+ # with test_app.test_client () as c:
+ # rv = c.get ('/?tequila=42')
+ # assert request.args ['tequila'] == '42'
+
+# import os
+# import flaskr
+# import unittest
+# import tempfile
+
+# class FlaskrTestCase(unittest.TestCase):
+
+# def setUp(self):
+# self.db_fd, flaskr.app.config['DATABASE'] = tempfile.mkstemp()
+# flaskr.app.config['TESTING'] = True
+# self.app = flaskr.app.test_client()
+# flaskr.init_db()
+
+# def tearDown(self):
+# os.close(self.db_fd)
+# os.unlink(flaskr.app.config['DATABASE'])
+
+# if __name__ == '__main__':
+# unittest.main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/utils/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/utils/__init__.py b/docker/distill/distill/utils/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/utils/exceptions.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/utils/exceptions.py b/docker/distill/distill/utils/exceptions.py
new file mode 100644
index 0000000..a391241
--- /dev/null
+++ b/docker/distill/distill/utils/exceptions.py
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class Error (Exception):
+ """Base class for exceptions."""
+ pass
+
+class ValidationError (Error):
+ """ Exceptions raised for errors in validated a url."""
+
+ def __init__ (self, url, msg):
+ self.url = url
+ self.msg = msg
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/utils/query_builder.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/utils/query_builder.py b/docker/distill/distill/utils/query_builder.py
new file mode 100644
index 0000000..017a08b
--- /dev/null
+++ b/docker/distill/distill/utils/query_builder.py
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class QueryBuilder (object):
+
+ def __init__ (self, query=None):
+ if query:
+ self.query = query
+ else:
+ self.query = {
+ "query" : {
+ "match_all" : {}
+ }
+ }
+
+
+ def add_filters (self, filters):
+ pass
+
+ def add_sorting (self, sort_field='', sort_order=''):
+ pass
+
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/utils/tests/__init__.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/utils/tests/__init__.py b/docker/distill/distill/utils/tests/__init__.py
new file mode 100644
index 0000000..09c5e2f
--- /dev/null
+++ b/docker/distill/distill/utils/tests/__init__.py
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''
+distill: tests module.
+
+Meant for use with py.test.
+Organize tests into files, each named xxx_test.py
+Read more here: http://pytest.org/
+'''
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/utils/validation.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/utils/validation.py b/docker/distill/distill/utils/validation.py
new file mode 100644
index 0000000..7cd3362
--- /dev/null
+++ b/docker/distill/distill/utils/validation.py
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from distill.utils.exceptions import ValidationError
+
+def validate_request (q):
+ """
+ Parse out request message and validate inputs
+
+ :param q: Url query string
+ :raises ValidationError: if the query is missing required parameters
+ """
+ if 'q' not in q:
+ raise ValidationError ("Missing required parameter: %s" % 'q')
+ else:
+ # Handle rest of parsing
+ pass
+
+def str2bool (v):
+ """
+ Convert string expression to boolean
+
+ :param v: Input value
+ :returns: Converted message as boolean type
+ :rtype: bool
+ """
+ return v.lower() in ("yes", "true", "t", "1")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/version.py
----------------------------------------------------------------------
diff --git a/docker/distill/distill/version.py b/docker/distill/distill/version.py
new file mode 100644
index 0000000..6532ea7
--- /dev/null
+++ b/docker/distill/distill/version.py
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Version information for Distill.
+
+This file is imported by ``Distill.__init__``,
+and parsed by ``setup.py``.
+"""
+
+__version__ = "0.1.3"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/requirements.txt
----------------------------------------------------------------------
diff --git a/docker/distill/requirements.txt b/docker/distill/requirements.txt
new file mode 100644
index 0000000..5137ad9
--- /dev/null
+++ b/docker/distill/requirements.txt
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Flask==0.10.1
+#networkx==1.11
+elasticsearch-dsl==2.0.0
+#numpy>=1.10.0
+#scipy>=0.17.0
+pandas>=0.18.1
+pytest>=3.0.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/setup.cfg
----------------------------------------------------------------------
diff --git a/docker/distill/setup.cfg b/docker/distill/setup.cfg
new file mode 100644
index 0000000..08020f4
--- /dev/null
+++ b/docker/distill/setup.cfg
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[egg_info]
+tag_build = 0.1.4
+tag_svn_revision = false
+
+[aliases]
+test=pytest
+
+[tool:pytest]
+addopts = --verbose --ignore=build --ignore=setup.py --ignore=dist --junitxml=test-report.xml --cov-report xml --cov=distill distill/.
+norecursedirs = *.eggs *env* .git
+
+[build_sphinx]
+source-dir = docs
+build-dir = docs/_build
+all_files = 1
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/setup.py
----------------------------------------------------------------------
diff --git a/docker/distill/setup.py b/docker/distill/setup.py
new file mode 100644
index 0000000..8ddd32f
--- /dev/null
+++ b/docker/distill/setup.py
@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from setuptools import setup, find_packages
+import distutils.cmd
+import distutils.log
+from setuptools.command.test import test as TestCommand
+import io, os, sys, subprocess
+
+if sys.version_info[:2] < (2, 7):
+ m = "Python 2.7 or later is required for Distill (%d.%d detected)."
+ raise ImportError (m % sys.version_info[:2])
+
+if sys.argv[-1] == 'setup.py':
+ print ("To install, run 'python setup.py install'")
+ print ()
+
+def read (*filenames, **kwargs):
+ encoding = kwargs.get ('encoding', 'utf-8')
+ sep = kwargs.get ('sep', '\n')
+ buf = []
+ for filename in filenames:
+ with io.open (filename, encoding=encoding) as f:
+ buf.append (f.read ())
+ return sep.join (buf)
+
+# Get the version string
+def get_version ():
+ basedir = os.path.dirname (__file__)
+ with open (os.path.join (basedir, 'distill/version.py')) as f:
+ version = {}
+ exec (f.read (), version)
+ return version['__version__']
+ raise RuntimeError ('No version info found.')
+
+setup (
+ name = "Distill",
+ version = get_version (),
+ url = "https://github.com/apache/incubator-senssoft-distill",
+ license = "Apache Software License",
+ author = "Michelle Beard",
+ author_email = "msbeard@apache.org",
+ description = "An analytical framework for UserALE.",
+ long_description = __doc__,
+ classifiers = [
+ 'Development Status :: 4 - Beta',
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 2.7',
+ 'Natural Language :: English',
+ 'Environment :: Web Environment',
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: Apache Software License',
+ 'Operating System :: OS Independent',
+ 'Private :: Do Not Upload"'
+ ],
+ keywords = "stout userale tap distill", # Separate with spaces
+ packages = find_packages (exclude=['examples', 'tests']),
+ include_package_data = True,
+ zip_safe = False,
+ setup_requires = ['pytest-runner'],
+ tests_require = ['pytest>=3.0.0', 'pytest-pylint', 'coverage'],
+ install_requires = ['Flask==0.10.1',
+ #'networkx==1.11',
+ 'elasticsearch-dsl==2.0.0',
+ #'numpy>=1.10.0',
+ #'scipy>=0.17.0',
+ 'pandas>=0.18.1'
+ ],
+ entry_points = {
+ 'console_scripts': [
+ 'dev = distill.server:dev_server'
+ ]
+ }
+)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/docker-compose.yml
----------------------------------------------------------------------
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
new file mode 100644
index 0000000..97ee0cb
--- /dev/null
+++ b/docker/docker-compose.yml
@@ -0,0 +1,78 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Apache TAP Production Build
+
+version: "2"
+services:
+ # ELK Stack
+ elasticsearch:
+ build: ./es
+ container_name: senssoft-elastic
+ ports:
+ - 9200:9200
+ - 9300:9300
+ volumes:
+ - ./es/data:/usr/share/elasticsearch/data
+ - ./es/logs:/usr/share/elasticsearch/logs
+ - ./es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+ environment:
+ ES_JAVA_OPTS: "-Xms1g -Xmx1g"
+ kibana:
+ build: ./kibana
+ container_name: senssoft-kibana
+ ports:
+ - 5601:5601
+ depends_on:
+ - "elasticsearch"
+ userale-forwarder:
+ build: ./logstash
+ container_name: senssoft-userale-forwarder
+ command: -f /etc/logstash/conf.d
+ ports:
+ - 8000:8000
+ volumes:
+ - ./logstash/config/logstash-userale.conf:/etc/logstash/conf.d/logstash-userale.conf
+ - ./logstash/templates/userale.json:/usr/share/logstash/templates/userale.json
+ depends_on:
+ - "elasticsearch"
+ environment:
+ LS_HEAP_SIZE: "2048m"
+ distill:
+ build:
+ context: .
+ dockerfile: ./distill/Dockerfile
+ container_name: distill
+ ports:
+ - 8090:8090
+ depends_on:
+ - elasticsearch
+ links:
+ - elasticsearch
+ db:
+ container_name: tap-db
+ build: ./db
+ ports:
+ - "5432:5432"
+ tap:
+ container_name: tap-web
+ build: ./tap
+ command: bash -c "python manage.py migrate && python manage.py runserver 0.0.0.0:8000"
+ ports:
+ - "8000:8000"
+ depends_on:
+ - db
+ links:
+ - db:db
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/es/._elasticsearch.yml
----------------------------------------------------------------------
diff --git a/docker/es/._elasticsearch.yml b/docker/es/._elasticsearch.yml
new file mode 100644
index 0000000..6da966a
Binary files /dev/null and b/docker/es/._elasticsearch.yml differ
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/es/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/es/Dockerfile b/docker/es/Dockerfile
new file mode 100644
index 0000000..1eb4748
--- /dev/null
+++ b/docker/es/Dockerfile
@@ -0,0 +1,5 @@
+FROM elasticsearch:5
+MAINTAINER Michelle Beard <ms...@apache.org>
+
+# Install XPack
+RUN elasticsearch-plugin install --batch x-pack
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/es/elasticsearch.yml
----------------------------------------------------------------------
diff --git a/docker/es/elasticsearch.yml b/docker/es/elasticsearch.yml
new file mode 100644
index 0000000..c795b7c
--- /dev/null
+++ b/docker/es/elasticsearch.yml
@@ -0,0 +1,98 @@
+# ======================== Elasticsearch Configuration =========================
+#
+# NOTE: Elasticsearch comes with reasonable defaults for most settings.
+# Before you set out to tweak and tune the configuration, make sure you
+# understand what are you trying to accomplish and the consequences.
+#
+# The primary way of configuring a node is via this file. This template lists
+# the most important settings you may want to configure for a production cluster.
+#
+# Please see the documentation for further information on configuration options:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration.html>
+#
+# ---------------------------------- Cluster -----------------------------------
+#
+# Use a descriptive name for your cluster:
+#
+cluster.name: SensSoft
+#
+# ------------------------------------ Node ------------------------------------
+#
+# Use a descriptive name for the node:
+#
+node.name: soft-01
+#
+# Add custom attributes to the node:
+#
+# node.rack: r1
+#
+# ----------------------------------- Paths ------------------------------------
+#
+# Path to directory where to store the data (separate multiple locations by comma):
+#
+path.data: /usr/share/elasticsearch/data
+#
+# Path to log files:
+#
+path.logs: /usr/share/elasticsearch/logs
+#
+#
+# ----------------------------------- Memory -----------------------------------
+#
+# Lock the memory on startup:
+#
+# bootstrap.mlockall: true
+#
+# Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory
+# available on the system and that the owner of the process is allowed to use this limit.
+#
+# Elasticsearch performs poorly when the system is swapping the memory.
+#
+# ---------------------------------- Network -----------------------------------
+#
+# Set the bind address to a specific IP (IPv4 or IPv6):
+#
+network.host: 0.0.0.0
+#
+# Set a custom port for HTTP:
+#
+# http.port: 9200
+#
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html>
+#
+# --------------------------------- Discovery ----------------------------------
+#
+# Pass an initial list of hosts to perform discovery when new node is started:
+# The default list of hosts is ["127.0.0.1", "[::1]"]
+#
+# discovery.zen.ping.unicast.hosts: ["host1", "host2"]
+#
+# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
+#
+discovery.zen.minimum_master_nodes: 1
+#
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html>
+#
+# ---------------------------------- Gateway -----------------------------------
+#
+# Block initial recovery after a full cluster restart until N nodes are started:
+#
+# gateway.recover_after_nodes: 3
+#
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html>
+#
+# ---------------------------------- Various -----------------------------------
+#
+# Disable starting multiple nodes on a single system:
+#
+# node.max_local_storage_nodes: 1
+#
+# Require explicit names when deleting indices:
+#
+# action.destructive_requires_name: true
+
+http.cors.enabled : true
+http.cors.allow-origin : "*"
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/kibana/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile
new file mode 100644
index 0000000..4814f47
--- /dev/null
+++ b/docker/kibana/Dockerfile
@@ -0,0 +1,7 @@
+FROM kibana:5
+MAINTAINER Michelle Beard <ms...@apache.org>
+
+# Install XPack
+RUN kibana-plugin install x-pack
+
+#CMD ["/tmp/entrypoint.sh"]
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/kibana/entrypoint.sh
----------------------------------------------------------------------
diff --git a/docker/kibana/entrypoint.sh b/docker/kibana/entrypoint.sh
new file mode 100644
index 0000000..c08d70a
--- /dev/null
+++ b/docker/kibana/entrypoint.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# Wait for the Elasticsearch container to be ready before starting Kibana.
+echo "Stalling for Elasticsearch"
+while true; do
+ nc -q 1 elasticsearch 9200 2>/dev/null && break
+done
+
+echo "Starting Kibana"
+exec kibana
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/logstash/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/logstash/Dockerfile b/docker/logstash/Dockerfile
new file mode 100644
index 0000000..3a9533a
--- /dev/null
+++ b/docker/logstash/Dockerfile
@@ -0,0 +1,11 @@
+FROM logstash:5
+MAINTAINER Michelle Beard <ms...@apache.org>
+
+# Get GeoIP DB
+RUN set -x \
+ && wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz \
+ && gunzip GeoIP.dat.gz \
+ && mkdir -p /usr/share/logstash/GeoIP \
+ && mv GeoIP.dat /usr/share/logstash/GeoIP/GeoIP.dat
+
+RUN mkdir /usr/share/logstash/templates
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/logstash/config/logstash-apache.conf
----------------------------------------------------------------------
diff --git a/docker/logstash/config/logstash-apache.conf b/docker/logstash/config/logstash-apache.conf
new file mode 100644
index 0000000..c95882b
--- /dev/null
+++ b/docker/logstash/config/logstash-apache.conf
@@ -0,0 +1,82 @@
+input {
+ # file {
+ # path => "/var/log/server_logs/access.log"
+ # start_position => "beginning"
+ # sincedb_path => "/dev/null"
+ # }
+
+ file {
+ path => [ "/var/log/server_logs/access.log" ]
+ start_position => "beginning"
+ sincedb_path => "/dev/null"
+ type => "apache-access"
+ }
+ # file {
+ # path => "/var/log/server_logs/error.log"
+ # start_position => "beginning"
+ # sincedb_path => "/dev/null"
+ # type => "apache-error"
+ # }
+}
+
+filter {
+ # ------------------------ Parse services logs into fields ---------------------------
+ # APACHE 2
+ if [type] == "apache-access" {
+ # To process log data (message's content) using some regex or precompiled GROK pattern
+ grok {
+ match => [ "message", "%{COMBINEDAPACHELOG}"]
+ }
+ # To extract log's time according to a date pattern
+ date {
+ match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z"]
+ }
+ # Extract browser information, if available.
+ if [agent] != "" {
+ useragent {
+ source => "agent"
+ }
+ }
+ # Extract client ip information, if available.
+ # if [clientip] != "" {
+ # geoip {
+ # source => "clientip"
+ # database => "/usr/share/logstash/GeoIP/GeoIP.dat"
+ # target => "apache_clientip"
+ # add_tag => [ "geoip" ]
+ # }
+ # }
+ }
+
+ # if [type] == "apache-error" {
+ # grok {
+ # match => [ "message", "%{APACHEERRORLOG}"]
+ # # Directory where to find the custom patterns
+ # patterns_dir => ["/etc/logstash/grok"]
+ # }
+ # if [clientip] != "" {
+ # geoip {
+ # source => "clientip"
+ # target => "apache_clientip"
+ # add_tag => [ "geoip" ]
+ # }
+ # }
+ # }
+}
+
+output {
+ # Output data to Elasticsearch instance
+ elasticsearch {
+ hosts => "elasticsearch:9200"
+ index => "apache"
+ user => "elastic"
+ password => "changeme"
+ manage_template => true
+ template_overwrite => true
+ template => "/usr/share/logstash/templates/apache.json"
+ template_name => "apache"
+ }
+
+ # Debug
+ stdout { codec => rubydebug }
+}