You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@senssoft.apache.org by ms...@apache.org on 2016/09/19 21:21:21 UTC
[3/5] incubator-senssoft-distill git commit: SENSSOFT-55 #Added
Docker support.
SENSSOFT-55 #Added Docker support.
Project: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/commit/e8c71a1b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/tree/e8c71a1b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/diff/e8c71a1b
Branch: refs/heads/master
Commit: e8c71a1b0fe6f959b986a8cd016a3ca26b38c97e
Parents: fb6c43a
Author: mooshu1x2 <mb...@draper.com>
Authored: Mon Sep 19 17:10:45 2016 -0400
Committer: mooshu1x2 <mb...@draper.com>
Committed: Mon Sep 19 17:10:45 2016 -0400
----------------------------------------------------------------------
.dockerignore | 26 ++++++
distill/config.cfg | 2 +-
distill/deploy/run_server.py | 29 -------
distill/server.py | 29 +++++++
docker-compose.yml | 65 ++++++++++++++
docker/distill/Dockerfile | 35 ++++++++
docker/distill/distill.conf | 29 +++++++
docker/es/Dockerfile | 62 ++++++++++++++
docker/es/elasticsearch.yml | 94 ++++++++++++++++++++
docker/es/logging.yml | 85 ++++++++++++++++++
docker/kibana/Dockerfile | 60 +++++++++++++
docker/kibana/entrypoint.sh | 24 ++++++
docker/kibana/kibana.yml | 79 +++++++++++++++++
docker/logstash/Dockerfile | 46 ++++++++++
docker/logstash/logstash.conf | 69 +++++++++++++++
docker/logstash/userale.json | 152 +++++++++++++++++++++++++++++++++
docker/webserver/Dockerfile | 34 ++++++++
docker/webserver/gunicorn.cfg | 27 ++++++
docker/webserver/nginx.conf | 57 +++++++++++++
docker/webserver/supervisord.conf | 13 +++
setup.py | 2 +-
21 files changed, 988 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/.dockerignore
----------------------------------------------------------------------
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..e2dfc86
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*.rst
+*.in
+LICENSE
+NOTICE
+env
+userale
+es
+docs
+doc_requirements.txt
+docker
+docker-compose.yml
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/distill/config.cfg
----------------------------------------------------------------------
diff --git a/distill/config.cfg b/distill/config.cfg
index b1fdfa7..189e3ea 100644
--- a/distill/config.cfg
+++ b/distill/config.cfg
@@ -30,7 +30,7 @@ MAPPINGS = '../path/to/mappings/MOT_Mappings.csv'
SELECTED = '../path/to/stout/selected_vars_for_distill.csv'
# Elasticsearch Configuration
-ES_HOST = 'http://localhost'
+ES_HOST = 'http://elasticsearch'
ES_PORT = 9200
HTTP_AUTH = None
USE_SSL = False
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/distill/deploy/run_server.py
----------------------------------------------------------------------
diff --git a/distill/deploy/run_server.py b/distill/deploy/run_server.py
deleted file mode 100644
index 23acd83..0000000
--- a/distill/deploy/run_server.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from distill import app
-from distill.app import *
-
-"""
-Start up a local WSGI server called development
-"""
-def dev_server ():
- host = app.config ['HOST']
- port = app.config ['PORT']
- debug = app.config ['DEBUG']
- app.run (host=host, port=port, debug=debug)
-
-if __name__ == '__main__':
- dev_server ()
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/distill/server.py
----------------------------------------------------------------------
diff --git a/distill/server.py b/distill/server.py
new file mode 100644
index 0000000..23acd83
--- /dev/null
+++ b/distill/server.py
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from distill import app
+from distill.app import *
+
+"""
+Start up a local WSGI server called development
+"""
+def dev_server ():
+ host = app.config ['HOST']
+ port = app.config ['PORT']
+ debug = app.config ['DEBUG']
+ app.run (host=host, port=port, debug=debug)
+
+if __name__ == '__main__':
+ dev_server ()
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker-compose.yml
----------------------------------------------------------------------
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..d546c6f
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Apache Distill Production Docker build
+
+version: "2"
+services:
+ distill:
+ build:
+ context: .
+ dockerfile: ./docker/distill/Dockerfile
+ container_name: distill
+ ports:
+ - 8090:8090
+ depends_on:
+ - elasticsearch
+ links:
+ - elasticsearch
+ # Web server
+ #webserver:
+ # build: ./docker/webserver
+ # container_name: webserver
+ # links:
+ # - distill
+ # ELK Stack
+ elasticsearch:
+ build: ./docker/es
+ container_name: elastic
+ ports:
+ - 9200:9200
+ - 9300:9300
+ volumes:
+ - ./es/data:/usr/share/elasticsearch/data
+ - ./es/logs:/usr/share/elasticsearch/logs
+ kibana:
+ build: ./docker/kibana
+ container_name: kibana
+ ports:
+ - 5601:5601
+ depends_on:
+ - elasticsearch
+ links:
+ - elasticsearch
+ logstash:
+ build: ./docker/logstash
+ container_name: logstash
+ command: logstash -f /etc/logstash/conf.d/logstash.conf
+ volumes:
+ - ./userale:/var/log/sennsoft
+ depends_on:
+ - elasticsearch
+ links:
+ - elasticsearch
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/distill/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/distill/Dockerfile b/docker/distill/Dockerfile
new file mode 100644
index 0000000..807ff3a
--- /dev/null
+++ b/docker/distill/Dockerfile
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:2
+
+# install system wide deps
+RUN apt-get -yqq update
+
+# Add code
+WORKDIR /app
+
+ADD ./requirements.txt /app
+
+# Install Distill dependencies
+RUN pip install -r requirements.txt
+
+# Expose Ports
+EXPOSE 8090
+
+# Rest
+ADD . /app
+RUN python setup.py develop
+CMD python distill/server.py
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/distill/distill.conf
----------------------------------------------------------------------
diff --git a/docker/distill/distill.conf b/docker/distill/distill.conf
new file mode 100644
index 0000000..a113dcb
--- /dev/null
+++ b/docker/distill/distill.conf
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+description "Gunicorn application server running Apache Distill"
+
+# Restart process if it ever fails
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+#Setup user and group that Gunicorn should be run as
+setuid nobody
+setgid www-data
+
+# Path to run_server
+chdir /path/to/distill
+exec gunicorn -c "gunicorn.cfg" scripts/run_server:app
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/es/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/es/Dockerfile b/docker/es/Dockerfile
new file mode 100644
index 0000000..b84b637
--- /dev/null
+++ b/docker/es/Dockerfile
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Ubuntu Xenial 16.04 [LTS]
+FROM ubuntu:16.04
+
+# install system wide deps
+RUN apt-get -yqq update
+RUN apt-get -yqq install openjdk-8-jre
+
+RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4
+
+ENV ELASTICSEARCH_VERSION 2.3.5
+ENV ELASTICSEARCH_REPO_BASE http://packages.elasticsearch.org/elasticsearch/2.x/debian
+
+RUN echo "deb $ELASTICSEARCH_REPO_BASE stable main" > /etc/apt/sources.list.d/elasticsearch.list
+
+# install elasticsearch
+RUN set -x \
+ && apt-get -yqq update \
+ && apt-get -yqq install --no-install-recommends elasticsearch=$ELASTICSEARCH_VERSION \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV PATH /usr/share/elasticsearch/bin:$PATH
+
+WORKDIR /usr/share/elasticsearch
+
+RUN set -ex \
+ && for path in \
+ ./data \
+ ./logs \
+ ./config \
+ ./config/scripts \
+ ; do \
+ mkdir -p "$path"; \
+ chown -R elasticsearch:elasticsearch "$path"; \
+ done
+
+COPY elasticsearch.yml /usr/share/elasticsearch/config
+COPY logging.yml /usr/share/elasticsearch/config
+
+USER elasticsearch
+
+# Install Elastic-HQ
+RUN plugin install royrusso/elasticsearch-HQ/2.0.3
+
+CMD ["elasticsearch"]
+
+# Run on ports 9200 & 9300
+EXPOSE 9200 9300
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/es/elasticsearch.yml
----------------------------------------------------------------------
diff --git a/docker/es/elasticsearch.yml b/docker/es/elasticsearch.yml
new file mode 100644
index 0000000..6126feb
--- /dev/null
+++ b/docker/es/elasticsearch.yml
@@ -0,0 +1,94 @@
+# ======================== Elasticsearch Configuration =========================
+#
+# NOTE: Elasticsearch comes with reasonable defaults for most settings.
+# Before you set out to tweak and tune the configuration, make sure you
+# understand what are you trying to accomplish and the consequences.
+#
+# The primary way of configuring a node is via this file. This template lists
+# the most important settings you may want to configure for a production cluster.
+#
+# Please see the documentation for further information on configuration options:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration.html>
+#
+# ---------------------------------- Cluster -----------------------------------
+#
+# Use a descriptive name for your cluster:
+#
+cluster.name: SensSoft
+#
+# ------------------------------------ Node ------------------------------------
+#
+# Use a descriptive name for the node:
+#
+node.name: soft-01
+#
+# Add custom attributes to the node:
+#
+# node.rack: r1
+#
+# ----------------------------------- Paths ------------------------------------
+#
+# Path to directory where to store the data (separate multiple locations by comma):
+#
+path.data: /usr/share/elasticsearch/data
+#
+# Path to log files:
+#
+path.logs: /usr/share/elasticsearch/logs
+#
+# ----------------------------------- Memory -----------------------------------
+#
+# Lock the memory on startup:
+#
+# bootstrap.mlockall: true
+#
+# Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory
+# available on the system and that the owner of the process is allowed to use this limit.
+#
+# Elasticsearch performs poorly when the system is swapping the memory.
+#
+# ---------------------------------- Network -----------------------------------
+#
+# Set the bind address to a specific IP (IPv4 or IPv6):
+#
+network.host: 0.0.0.0
+#
+# Set a custom port for HTTP:
+#
+# http.port: 9200
+#
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html>
+#
+# --------------------------------- Discovery ----------------------------------
+#
+# Pass an initial list of hosts to perform discovery when new node is started:
+# The default list of hosts is ["127.0.0.1", "[::1]"]
+#
+# discovery.zen.ping.unicast.hosts: ["host1", "host2"]
+#
+# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
+#
+# discovery.zen.minimum_master_nodes: 3
+#
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html>
+#
+# ---------------------------------- Gateway -----------------------------------
+#
+# Block initial recovery after a full cluster restart until N nodes are started:
+#
+# gateway.recover_after_nodes: 3
+#
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html>
+#
+# ---------------------------------- Various -----------------------------------
+#
+# Disable starting multiple nodes on a single system:
+#
+# node.max_local_storage_nodes: 1
+#
+# Require explicit names when deleting indices:
+#
+# action.destructive_requires_name: true
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/es/logging.yml
----------------------------------------------------------------------
diff --git a/docker/es/logging.yml b/docker/es/logging.yml
new file mode 100644
index 0000000..939aa1e
--- /dev/null
+++ b/docker/es/logging.yml
@@ -0,0 +1,85 @@
+# you can override this using by setting a system property, for example -Des.logger.level=DEBUG
+es.logger.level: INFO
+rootLogger: ${es.logger.level}, console, file
+logger:
+ # log action execution errors for easier debugging
+ action: DEBUG
+
+ # deprecation logging, turn to DEBUG to see them
+ deprecation: INFO, deprecation_log_file
+
+ # reduce the logging for aws, too much is logged under the default INFO
+ com.amazonaws: WARN
+ # aws will try to do some sketchy JMX stuff, but its not needed.
+ com.amazonaws.jmx.SdkMBeanRegistrySupport: ERROR
+ com.amazonaws.metrics.AwsSdkMetrics: ERROR
+
+ org.apache.http: INFO
+
+ # gateway
+ #gateway: DEBUG
+ #index.gateway: DEBUG
+
+ # peer shard recovery
+ #indices.recovery: DEBUG
+
+ # discovery
+ #discovery: TRACE
+
+ index.search.slowlog: TRACE, index_search_slow_log_file
+ index.indexing.slowlog: TRACE, index_indexing_slow_log_file
+
+additivity:
+ index.search.slowlog: false
+ index.indexing.slowlog: false
+ deprecation: false
+
+appender:
+ console:
+ type: console
+ layout:
+ type: consolePattern
+ conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+ file:
+ type: dailyRollingFile
+ file: ${path.logs}/${cluster.name}.log
+ datePattern: "'.'yyyy-MM-dd"
+ layout:
+ type: pattern
+ conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %.10000m%n"
+
+ # Use the following log4j-extras RollingFileAppender to enable gzip compression of log files.
+ # For more information see https://logging.apache.org/log4j/extras/apidocs/org/apache/log4j/rolling/RollingFileAppender.html
+ #file:
+ #type: extrasRollingFile
+ #file: ${path.logs}/${cluster.name}.log
+ #rollingPolicy: timeBased
+ #rollingPolicy.FileNamePattern: ${path.logs}/${cluster.name}.log.%d{yyyy-MM-dd}.gz
+ #layout:
+ #type: pattern
+ #conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+ deprecation_log_file:
+ type: dailyRollingFile
+ file: ${path.logs}/${cluster.name}_deprecation.log
+ datePattern: "'.'yyyy-MM-dd"
+ layout:
+ type: pattern
+ conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+ index_search_slow_log_file:
+ type: dailyRollingFile
+ file: ${path.logs}/${cluster.name}_index_search_slowlog.log
+ datePattern: "'.'yyyy-MM-dd"
+ layout:
+ type: pattern
+ conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+ index_indexing_slow_log_file:
+ type: dailyRollingFile
+ file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log
+ datePattern: "'.'yyyy-MM-dd"
+ layout:
+ type: pattern
+ conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/kibana/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile
new file mode 100644
index 0000000..e37cfa0
--- /dev/null
+++ b/docker/kibana/Dockerfile
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Ubuntu Xenial 16.04 [LTS]
+FROM ubuntu:16.04
+
+# install system wide deps
+RUN apt-get -yqq update
+RUN apt-get -yqq install netcat
+
+RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4
+
+ENV KIBANA_VERSION 4.5.4
+ENV KIBANA_REPO_BASE http://packages.elasticsearch.org/kibana/4.5/debian
+
+RUN echo "deb $KIBANA_REPO_BASE stable main" > /etc/apt/sources.list.d/kibana.list
+
+# install kibana
+RUN set -x \
+ && apt-get -yqq update \
+ && apt-get -yqq install --no-install-recommends kibana=$KIBANA_VERSION \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV PATH /opt/kibana/bin:$PATH
+
+WORKDIR /opt/kibana
+
+RUN set -ex \
+ && for path in \
+ ./config \
+ ; do \
+ chown -R kibana:kibana "$path"; \
+ done
+
+COPY kibana.yml /opt/kibana/config
+
+COPY entrypoint.sh /tmp/entrypoint.sh
+RUN chmod +x /tmp/entrypoint.sh
+
+# Install Sense plugin
+RUN kibana plugin --install elastic/sense
+
+USER kibana
+
+CMD ["/tmp/entrypoint.sh"]
+
+# Run on port 5601
+EXPOSE 5601
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/kibana/entrypoint.sh
----------------------------------------------------------------------
diff --git a/docker/kibana/entrypoint.sh b/docker/kibana/entrypoint.sh
new file mode 100644
index 0000000..02bd91f
--- /dev/null
+++ b/docker/kibana/entrypoint.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Wait for the Elasticsearch container to be ready before starting Kibana.
+echo "Stalling for Elasticsearch"
+while true; do
+ nc -q 1 elasticsearch 9200 2>/dev/null && break
+done
+
+echo "Starting Kibana"
+exec kibana
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/kibana/kibana.yml
----------------------------------------------------------------------
diff --git a/docker/kibana/kibana.yml b/docker/kibana/kibana.yml
new file mode 100644
index 0000000..8bc978f
--- /dev/null
+++ b/docker/kibana/kibana.yml
@@ -0,0 +1,79 @@
+# Kibana is served by a back end server. This controls which port to use.
+server.port: 5601
+
+# The host to bind the server to.
+server.host: "0.0.0.0"
+
+# If you are running kibana behind a proxy, and want to mount it at a path,
+# specify that path here. The basePath can't end in a slash.
+# server.basePath: ""
+
+# The maximum payload size in bytes on incoming server requests.
+# server.maxPayloadBytes: 1048576
+
+# The Elasticsearch instance to use for all your queries.
+elasticsearch.url: "http://elasticsearch:9200"
+
+# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false,
+# then the host you use to connect to *this* Kibana instance will be sent.
+elasticsearch.preserveHost: true
+
+# Kibana uses an index in Elasticsearch to store saved searches, visualizations
+# and dashboards. It will create a new index if it doesn't already exist.
+kibana.index: ".kibana"
+
+# The default application to load.
+kibana.defaultAppId: "discover"
+
+# If your Elasticsearch is protected with basic auth, these are the user credentials
+# used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana
+# users will still need to authenticate with Elasticsearch (which is proxied through
+# the Kibana server)
+# elasticsearch.username: "user"
+# elasticsearch.password: "pass"
+
+# SSL for outgoing requests from the Kibana Server to the browser (PEM formatted)
+# server.ssl.cert: /path/to/your/server.crt
+# server.ssl.key: /path/to/your/server.key
+
+# Optional setting to validate that your Elasticsearch backend uses the same key files (PEM formatted)
+# elasticsearch.ssl.cert: /path/to/your/client.crt
+# elasticsearch.ssl.key: /path/to/your/client.key
+
+# If you need to provide a CA certificate for your Elasticsearch instance, put
+# the path of the pem file here.
+# elasticsearch.ssl.ca: /path/to/your/CA.pem
+
+# Set to false to have a complete disregard for the validity of the SSL
+# certificate.
+# elasticsearch.ssl.verify: true
+
+# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to
+# request_timeout setting
+# elasticsearch.pingTimeout: 1500
+
+# Time in milliseconds to wait for responses from the back end or elasticsearch.
+# This must be > 0
+# elasticsearch.requestTimeout: 30000
+
+# Time in milliseconds for Elasticsearch to wait for responses from shards.
+# Set to 0 to disable.
+# elasticsearch.shardTimeout: 0
+
+# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying
+# elasticsearch.startupTimeout: 5000
+
+# Set the path to where you would like the process id file to be created.
+# pid.file: /var/run/kibana.pid
+
+# If you would like to send the log output to a file you can set the path below.
+logging.dest: stdout
+
+# Set this to true to suppress all logging output.
+# logging.silent: false
+
+# Set this to true to suppress all logging output except for error messages.
+logging.quiet: true
+
+# Set this to true to log all events, including system usage information and all requests.
+# logging.verbose: false
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/logstash/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/logstash/Dockerfile b/docker/logstash/Dockerfile
new file mode 100644
index 0000000..bd5929b
--- /dev/null
+++ b/docker/logstash/Dockerfile
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Ubuntu Xenial 16.04 [LTS]
+FROM ubuntu:16.04
+
+# install system wide deps
+RUN apt-get -yqq update
+RUN apt-get -yqq install openjdk-8-jre
+RUN apt-get -yqq install wget
+RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4
+
+ENV LOGSTASH_VERSION 1:2.3.4-1
+ENV LOGSTASH_REPO_BASE http://packages.elasticsearch.org/logstash/2.3/debian
+
+RUN echo "deb $LOGSTASH_REPO_BASE stable main" > /etc/apt/sources.list.d/logstash.list
+
+# install logstash
+RUN set -x \
+ && apt-get -yqq update \
+ && apt-get -yqq install --no-install-recommends logstash=$LOGSTASH_VERSION \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV PATH /opt/logstash/bin:$PATH
+
+# Get GeoIP DB
+#RUN set -x \
+# && wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz \
+# && gunzip GeoIP.dat.gz \
+# && mkdir -p /usr/share/GeoIP \
+# && mv GeoIP.dat /usr/share/GeoIP
+
+ADD logstash.conf /etc/logstash/conf.d/logstash.conf
+ADD userale.json /etc/logstash/conf.d/userale.json
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/logstash/logstash.conf
----------------------------------------------------------------------
diff --git a/docker/logstash/logstash.conf b/docker/logstash/logstash.conf
new file mode 100644
index 0000000..09b5caf
--- /dev/null
+++ b/docker/logstash/logstash.conf
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+input {
+ file {
+ path => [
+ "/var/log/sennsoft/*.log"
+ ]
+ codec => "json"
+ start_position => "beginning"
+ sincedb_path => "/dev/null"
+ }
+}
+
+## Add your filters / logstash plugins configuration here
+filter {
+ mutate {
+ remove_field => [ "message", "@version", "@timestamp"]
+ }
+
+ grok {
+ match => [ "useraleVersion", "(?<major_ver>\d+).(?<minor_ver>\d+)(.(?<patch_ver>\d+))?" ]
+ match => [ "toolVersion", "(?<tool_major_ver>\d+).(?<tool_minor_ver>\d+)(.(?<tool_patch_ver>\d+))?" ]
+ }
+
+ #dns {
+ # add_field => [ "hostname", "%{host}" ]
+ #}
+
+ #dns {
+ # resolve => [ "host" ]
+ # action => [ "replace" ]
+ #}
+
+ #geoip {
+ # source => "host"
+ # database => "/usr/share/GeoIP/GeoIP.dat"
+ # target => "geoip"
+ #}
+}
+
+output {
+ # Output data to Elasticsearch instance
+ elasticsearch {
+ hosts => "elasticsearch:9200"
+ action => "index"
+ index => "userale"
+ document_type => "logs"
+ manage_template => true
+ template_overwrite => true
+ template => "/etc/logstash/conf.d/userale.json"
+ template_name => "userale"
+ }
+
+ # Debug
+ stdout { codec => rubydebug }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/logstash/userale.json
----------------------------------------------------------------------
diff --git a/docker/logstash/userale.json b/docker/logstash/userale.json
new file mode 100644
index 0000000..3d53d7b
--- /dev/null
+++ b/docker/logstash/userale.json
@@ -0,0 +1,152 @@
+{
+ "template" : "userale",
+ "order" : 1,
+ "settings" : {
+ "number_of_shards" : 1,
+ "number_of_replicas" : 0,
+ "index.refresh_interval" : "5s"
+ },
+ "mappings" : {
+ "_default_" : {
+ "_all" : {
+ "enabled" : false
+ },
+ "properties" : {
+ "geoip" : {
+ "properties" : {
+ "ip" : {
+ "type" : "ip"
+ },
+ "country_code2" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "country_code3" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "country_name" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "continent_code" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "region_name" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "city_name" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "postal_code" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "latitude" : {
+ "type" : "double"
+ },
+ "longitude" : {
+ "type" : "double"
+ },
+ "dma_code" : {
+ "type" : "integer"
+ },
+ "area_code" : {
+ "type" : "long"
+ },
+ "timezone" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "real_region_name" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "location" : {
+ "doc_values" : true,
+ "type" : "geo_point"
+ }
+ }
+ }
+ }
+ },
+ "logs" : {
+ "properties" : {
+ "userAction" : {
+ "type" : "boolean"
+ },
+ "type" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "clientTime" : {
+ "type" : "date",
+ "format" : "strict_date_optional_time||epoch_millis"
+ },
+ "target" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "path" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "details" : {
+ "type" : "object"
+ },
+ "location" : {
+ "type" : "object",
+ "properties" : {
+ "x" : {
+ "type" : "integer"
+ },
+ "y" : {
+ "type" : "integer"
+ }
+ }
+ },
+ "userId" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "session" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "toolName" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "toolVersion" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "useraleVersion" : {
+ "index" : "not_analyzed",
+ "type" : "string"
+ },
+ "major_ver" : {
+ "type" : "integer"
+ },
+ "minor_ver" : {
+ "type" : "integer"
+ },
+ "patch_ver" : {
+ "type" : "integer"
+ },
+ "tool_major_ver" : {
+ "type" : "integer"
+ },
+ "tool_minor_ver" : {
+ "type" : "integer"
+ },
+ "tool_patch_ver" : {
+ "type" : "integer"
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/webserver/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/webserver/Dockerfile b/docker/webserver/Dockerfile
new file mode 100644
index 0000000..99fc7a5
--- /dev/null
+++ b/docker/webserver/Dockerfile
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:2
+
+# install system wide deps
+RUN apt-get -yqq update
+RUN apt-get -yqq install nginx
+
+# Install Supervisord
+RUN pip install supervisor-stdout
+
+# Add configuration files
+ADD ./supervisord.conf /etc/supervisord.conf
+ADD ./gunicorn.cfg /etc/gunicorn.cfg
+ADD ./nginx.conf /etc/nginx/nginx.conf
+
+# restart nginx to load the config
+RUN service nginx stop
+
+# start supervisor to run our wsgi server
+CMD supervisord -c /etc/supervisord.conf -n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/webserver/gunicorn.cfg
----------------------------------------------------------------------
diff --git a/docker/webserver/gunicorn.cfg b/docker/webserver/gunicorn.cfg
new file mode 100644
index 0000000..5b199c4
--- /dev/null
+++ b/docker/webserver/gunicorn.cfg
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+gunicorn WSGI server configuration.
+"""
+
+import multiprocessing
+
+bind = ["unix:distill.sock"]
+workers = max (multiprocessing.cpu_count () * 2 + 1, 4)
+timeout = 10
+umask = 007
+accesslog = "-"
+errorlog = "-"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/webserver/nginx.conf
----------------------------------------------------------------------
diff --git a/docker/webserver/nginx.conf b/docker/webserver/nginx.conf
new file mode 100644
index 0000000..3a46ed8
--- /dev/null
+++ b/docker/webserver/nginx.conf
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+sudo /etc/init.d/nginx start
+sudo rm /etc/nginx/sites-enabled/default
+sudo touch /etc/nginx/sites-available/distill
+sudo ln -s /etc/nginx/sites-available/distill /etc/nginx/sites-enabled/distill
+
+sudo vim /etc/nginx/sites-enabled/distill
+
+user nobody;
+worker_processes 2;
+
+error_log /var/log/nginx/error.log;
+error_log /var/log/nginx/error.log notice;
+error_log /var/log/nginx/error.log info;
+
+# user nobody nogroup
+pid /tmp/nginx.pid;
+
+events {
+ worker_connections 1024;
+}
+
+http {
+ include mime.types;
+ default_type application/octet-stream;
+ access_log /var/log/nginx/access.log combined;
+ sendfile on;
+
+ server {
+ listen 80 default;
+ server_name localhost;
+ keepalive_timeout 5;
+
+ access_log /var/log/nginx/distill.log;
+
+ location / {
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $host;
+ proxy_pass http://127.0.0.1:8000;
+ }
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/webserver/supervisord.conf
----------------------------------------------------------------------
diff --git a/docker/webserver/supervisord.conf b/docker/webserver/supervisord.conf
new file mode 100644
index 0000000..bb15542
--- /dev/null
+++ b/docker/webserver/supervisord.conf
@@ -0,0 +1,13 @@
+[supervisord]
+nodaemon = true
+
+[program:nginx]
+command = /usr/sbin/nginx
+startsecs = 5
+stdout_events_enabled = true
+stderr_events_enabled = true
+
+[program:app-gunicorn]
+command = gunicorn -c "/etc/gunicorn.cfg" scripts/run_server:app
+stdout_events_enabled = true
+stderr_events_enabled = true
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/setup.py
----------------------------------------------------------------------
diff --git a/setup.py b/setup.py
index ccf0855..aa474d2 100644
--- a/setup.py
+++ b/setup.py
@@ -92,7 +92,7 @@ setup (
],
entry_points = {
'console_scripts': [
- 'dev = distill.deploy.run_server:dev_server'
+ 'dev = distill.server:dev_server'
]
}
)