You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@senssoft.apache.org by ms...@apache.org on 2016/12/06 22:59:49 UTC
incubator-senssoft-distill git commit: Updated ELK stack to 5.0
Repository: incubator-senssoft-distill
Updated Branches:
refs/heads/elk5.0-upgrade [created] 923a3052e
Updated ELK stack to 5.0
Project: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/commit/923a3052
Tree: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/tree/923a3052
Diff: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/diff/923a3052
Branch: refs/heads/elk5.0-upgrade
Commit: 923a3052e37028b93b7b68d94274fd6f4b146c12
Parents: a22b46c
Author: mooshu1x2 <mb...@draper.com>
Authored: Tue Dec 6 17:59:32 2016 -0500
Committer: mooshu1x2 <mb...@draper.com>
Committed: Tue Dec 6 17:59:32 2016 -0500
----------------------------------------------------------------------
docker-compose.yml | 22 +++++-----
docker/es/Dockerfile | 50 ++--------------------
docker/es/elasticsearch.yml | 3 ++
docker/es/logging.yml | 85 --------------------------------------
docker/kibana/Dockerfile | 43 +++----------------
docker/logstash/Dockerfile | 30 ++++----------
docker/logstash/logstash.conf | 2 +-
docker/logstash/userale.json | 79 ++++++++++++++++++-----------------
8 files changed, 73 insertions(+), 241 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker-compose.yml
----------------------------------------------------------------------
diff --git a/docker-compose.yml b/docker-compose.yml
index 52746b0..773151d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -21,11 +21,9 @@ services:
build:
context: .
dockerfile: ./docker/distill/Dockerfile
- container_name: distill
+ container_name: senssoft-distill
ports:
- 8090:8090
- depends_on:
- - elasticsearch
links:
- elasticsearch
# Web server
@@ -37,31 +35,31 @@ services:
# ELK Stack
elasticsearch:
build: ./docker/es
- container_name: elastic
+ container_name: senssoft-elastic
ports:
- 9200:9200
- 9300:9300
volumes:
- ./es/data:/usr/share/elasticsearch/data
- ./es/logs:/usr/share/elasticsearch/logs
+ - ./docker/es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+ environment:
+ ES_JAVA_OPTS: "-Xms1g -Xmx1g"
kibana:
build: ./docker/kibana
- container_name: kibana
+ container_name: senssoft-kibana
ports:
- 5601:5601
- depends_on:
- - elasticsearch
links:
- elasticsearch
logstash:
build: ./docker/logstash
- container_name: logstash
+ container_name: senssoft-logstash
ports:
- 8000:8000
- command: logstash -f /etc/logstash/conf.d/logstash.conf
volumes:
- - ./userale:/var/log/sennsoft
- depends_on:
- - elasticsearch
+ - ./userale:/var/log/senssoft
links:
- elasticsearch
+ environment:
+ LS_HEAP_SIZE: "2048m"
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker/es/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/es/Dockerfile b/docker/es/Dockerfile
index b84b637..8acde7c 100644
--- a/docker/es/Dockerfile
+++ b/docker/es/Dockerfile
@@ -13,50 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Ubuntu Xenial 16.04 [LTS]
-FROM ubuntu:16.04
+FROM elasticsearch:5
+MAINTAINER Michelle Beard <ms...@apache.org>
-# install system wide deps
-RUN apt-get -yqq update
-RUN apt-get -yqq install openjdk-8-jre
-
-RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4
-
-ENV ELASTICSEARCH_VERSION 2.3.5
-ENV ELASTICSEARCH_REPO_BASE http://packages.elasticsearch.org/elasticsearch/2.x/debian
-
-RUN echo "deb $ELASTICSEARCH_REPO_BASE stable main" > /etc/apt/sources.list.d/elasticsearch.list
-
-# install elasticsearch
-RUN set -x \
- && apt-get -yqq update \
- && apt-get -yqq install --no-install-recommends elasticsearch=$ELASTICSEARCH_VERSION \
- && rm -rf /var/lib/apt/lists/*
-
-ENV PATH /usr/share/elasticsearch/bin:$PATH
-
-WORKDIR /usr/share/elasticsearch
-
-RUN set -ex \
- && for path in \
- ./data \
- ./logs \
- ./config \
- ./config/scripts \
- ; do \
- mkdir -p "$path"; \
- chown -R elasticsearch:elasticsearch "$path"; \
- done
-
-COPY elasticsearch.yml /usr/share/elasticsearch/config
-COPY logging.yml /usr/share/elasticsearch/config
-
-USER elasticsearch
-
-# Install Elastic-HQ
-RUN plugin install royrusso/elasticsearch-HQ/2.0.3
-
-CMD ["elasticsearch"]
-
-# Run on ports 9200 & 9300
-EXPOSE 9200 9300
\ No newline at end of file
+# Install XPack
+# RUN elasticsearch-plugin install --batch x-pack
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker/es/elasticsearch.yml
----------------------------------------------------------------------
diff --git a/docker/es/elasticsearch.yml b/docker/es/elasticsearch.yml
index 6126feb..bac0b1d 100644
--- a/docker/es/elasticsearch.yml
+++ b/docker/es/elasticsearch.yml
@@ -92,3 +92,6 @@ network.host: 0.0.0.0
# Require explicit names when deleting indices:
#
# action.destructive_requires_name: true
+
+http.cors.enabled : true
+http.cors.allow-origin : "*"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker/es/logging.yml
----------------------------------------------------------------------
diff --git a/docker/es/logging.yml b/docker/es/logging.yml
deleted file mode 100644
index 939aa1e..0000000
--- a/docker/es/logging.yml
+++ /dev/null
@@ -1,85 +0,0 @@
-# you can override this using by setting a system property, for example -Des.logger.level=DEBUG
-es.logger.level: INFO
-rootLogger: ${es.logger.level}, console, file
-logger:
- # log action execution errors for easier debugging
- action: DEBUG
-
- # deprecation logging, turn to DEBUG to see them
- deprecation: INFO, deprecation_log_file
-
- # reduce the logging for aws, too much is logged under the default INFO
- com.amazonaws: WARN
- # aws will try to do some sketchy JMX stuff, but its not needed.
- com.amazonaws.jmx.SdkMBeanRegistrySupport: ERROR
- com.amazonaws.metrics.AwsSdkMetrics: ERROR
-
- org.apache.http: INFO
-
- # gateway
- #gateway: DEBUG
- #index.gateway: DEBUG
-
- # peer shard recovery
- #indices.recovery: DEBUG
-
- # discovery
- #discovery: TRACE
-
- index.search.slowlog: TRACE, index_search_slow_log_file
- index.indexing.slowlog: TRACE, index_indexing_slow_log_file
-
-additivity:
- index.search.slowlog: false
- index.indexing.slowlog: false
- deprecation: false
-
-appender:
- console:
- type: console
- layout:
- type: consolePattern
- conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
-
- file:
- type: dailyRollingFile
- file: ${path.logs}/${cluster.name}.log
- datePattern: "'.'yyyy-MM-dd"
- layout:
- type: pattern
- conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %.10000m%n"
-
- # Use the following log4j-extras RollingFileAppender to enable gzip compression of log files.
- # For more information see https://logging.apache.org/log4j/extras/apidocs/org/apache/log4j/rolling/RollingFileAppender.html
- #file:
- #type: extrasRollingFile
- #file: ${path.logs}/${cluster.name}.log
- #rollingPolicy: timeBased
- #rollingPolicy.FileNamePattern: ${path.logs}/${cluster.name}.log.%d{yyyy-MM-dd}.gz
- #layout:
- #type: pattern
- #conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
-
- deprecation_log_file:
- type: dailyRollingFile
- file: ${path.logs}/${cluster.name}_deprecation.log
- datePattern: "'.'yyyy-MM-dd"
- layout:
- type: pattern
- conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
-
- index_search_slow_log_file:
- type: dailyRollingFile
- file: ${path.logs}/${cluster.name}_index_search_slowlog.log
- datePattern: "'.'yyyy-MM-dd"
- layout:
- type: pattern
- conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
-
- index_indexing_slow_log_file:
- type: dailyRollingFile
- file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log
- datePattern: "'.'yyyy-MM-dd"
- layout:
- type: pattern
- conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker/kibana/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile
index e37cfa0..0febc73 100644
--- a/docker/kibana/Dockerfile
+++ b/docker/kibana/Dockerfile
@@ -13,48 +13,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Ubuntu Xenial 16.04 [LTS]
-FROM ubuntu:16.04
+FROM kibana:5
+MAINTAINER Michelle Beard <ms...@apache.org>
-# install system wide deps
-RUN apt-get -yqq update
-RUN apt-get -yqq install netcat
-
-RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4
-
-ENV KIBANA_VERSION 4.5.4
-ENV KIBANA_REPO_BASE http://packages.elasticsearch.org/kibana/4.5/debian
-
-RUN echo "deb $KIBANA_REPO_BASE stable main" > /etc/apt/sources.list.d/kibana.list
-
-# install kibana
-RUN set -x \
- && apt-get -yqq update \
- && apt-get -yqq install --no-install-recommends kibana=$KIBANA_VERSION \
- && rm -rf /var/lib/apt/lists/*
-
-ENV PATH /opt/kibana/bin:$PATH
-
-WORKDIR /opt/kibana
-
-RUN set -ex \
- && for path in \
- ./config \
- ; do \
- chown -R kibana:kibana "$path"; \
- done
-
-COPY kibana.yml /opt/kibana/config
+RUN apt-get update && apt-get install -y netcat bzip2
COPY entrypoint.sh /tmp/entrypoint.sh
RUN chmod +x /tmp/entrypoint.sh
-# Install Sense plugin
-RUN kibana plugin --install elastic/sense
-
-USER kibana
+# Install XPack
+# RUN kibana-plugin install x-pack
CMD ["/tmp/entrypoint.sh"]
-
-# Run on port 5601
-EXPOSE 5601
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker/logstash/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/logstash/Dockerfile b/docker/logstash/Dockerfile
index bd5929b..c38251b 100644
--- a/docker/logstash/Dockerfile
+++ b/docker/logstash/Dockerfile
@@ -13,27 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Ubuntu Xenial 16.04 [LTS]
-FROM ubuntu:16.04
+FROM logstash:5
+MAINTAINER Michelle Beard <ms...@apache.org>
-# install system wide deps
-RUN apt-get -yqq update
-RUN apt-get -yqq install openjdk-8-jre
-RUN apt-get -yqq install wget
-RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4
-
-ENV LOGSTASH_VERSION 1:2.3.4-1
-ENV LOGSTASH_REPO_BASE http://packages.elasticsearch.org/logstash/2.3/debian
-
-RUN echo "deb $LOGSTASH_REPO_BASE stable main" > /etc/apt/sources.list.d/logstash.list
-
-# install logstash
-RUN set -x \
- && apt-get -yqq update \
- && apt-get -yqq install --no-install-recommends logstash=$LOGSTASH_VERSION \
- && rm -rf /var/lib/apt/lists/*
-
-ENV PATH /opt/logstash/bin:$PATH
# Get GeoIP DB
#RUN set -x \
@@ -42,5 +24,9 @@ ENV PATH /opt/logstash/bin:$PATH
# && mkdir -p /usr/share/GeoIP \
# && mv GeoIP.dat /usr/share/GeoIP
-ADD logstash.conf /etc/logstash/conf.d/logstash.conf
-ADD userale.json /etc/logstash/conf.d/userale.json
\ No newline at end of file
+# Setup Logstash
+COPY logstash.conf /etc/logstash/conf.d/logstash.conf
+COPY userale.json /etc/logstash/conf.d/userale.json
+
+# Run Logstash
+CMD ["logstash", "-f", "/etc/logstash/conf.d/logstash.conf"]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker/logstash/logstash.conf
----------------------------------------------------------------------
diff --git a/docker/logstash/logstash.conf b/docker/logstash/logstash.conf
index ca831f5..375f3b9 100644
--- a/docker/logstash/logstash.conf
+++ b/docker/logstash/logstash.conf
@@ -25,7 +25,7 @@ input {
file {
path => [
- "/var/log/sennsoft/*.log"
+ "/var/log/senssoft/*.log"
]
codec => "json"
start_position => "beginning"
http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker/logstash/userale.json
----------------------------------------------------------------------
diff --git a/docker/logstash/userale.json b/docker/logstash/userale.json
index 3d53d7b..e5d8f91 100644
--- a/docker/logstash/userale.json
+++ b/docker/logstash/userale.json
@@ -18,38 +18,38 @@
"type" : "ip"
},
"country_code2" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"country_code3" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"country_name" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"continent_code" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"region_name" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"city_name" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"postal_code" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"latitude" : {
- "type" : "double"
+ "type" : "float"
},
"longitude" : {
- "type" : "double"
+ "type" : "float"
},
"dma_code" : {
"type" : "integer"
@@ -58,12 +58,12 @@
"type" : "long"
},
"timezone" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"real_region_name" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"location" : {
"doc_values" : true,
@@ -79,23 +79,28 @@
"type" : "boolean"
},
"type" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"clientTime" : {
"type" : "date",
"format" : "strict_date_optional_time||epoch_millis"
},
"target" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"path" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"details" : {
- "type" : "object"
+ "type" : "object",
+ "properties" : {
+ "source" : {
+ "type" : "keyword"
+ }
+ }
},
"location" : {
"type" : "object",
@@ -109,24 +114,24 @@
}
},
"userId" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"session" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"toolName" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"toolVersion" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"useraleVersion" : {
- "index" : "not_analyzed",
- "type" : "string"
+
+ "type" : "keyword"
},
"major_ver" : {
"type" : "integer"