You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/06/14 19:35:23 UTC
incubator-metron git commit: METRON-212: Allow additional
Elasticsearch templates to be loaded to the index (dlyle65535 via cestella)
closes apache/incubator-metron#145
Repository: incubator-metron
Updated Branches:
refs/heads/master 363ca91a4 -> 739e2eb52
METRON-212: Allow additional Elasticsearch templates to be loaded to the index (dlyle65535 via cestella) closes apache/incubator-metron#145
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/739e2eb5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/739e2eb5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/739e2eb5
Branch: refs/heads/master
Commit: 739e2eb523dd6b4daeeccd3bab5a4a614ace8328
Parents: 363ca91
Author: dlyle65535 <dl...@gmail.com>
Authored: Tue Jun 14 15:35:15 2016 -0400
Committer: cstella <ce...@gmail.com>
Committed: Tue Jun 14 15:35:15 2016 -0400
----------------------------------------------------------------------
metron-deployment/playbooks/metron_install.yml | 5 +-
.../elasticsearch/files/bro_index.template | 12 ---
.../elasticsearch/files/snort_index.template | 19 -----
.../elasticsearch/files/yaf_index.template | 85 --------------------
.../elasticsearch/tasks/create-indices.yml | 51 ------------
.../roles/elasticsearch/tasks/main.yml | 3 -
.../files/es_templates/bro_index.template | 12 +++
.../files/es_templates/snort_index.template | 19 +++++
.../files/es_templates/yaf_index.template | 85 ++++++++++++++++++++
.../tasks/load_templates.yml | 45 +++++++++++
.../tasks/main.yml | 19 +++++
11 files changed, 183 insertions(+), 172 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/739e2eb5/metron-deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml
index 7435b97..cdeac82 100644
--- a/metron-deployment/playbooks/metron_install.yml
+++ b/metron-deployment/playbooks/metron_install.yml
@@ -48,6 +48,7 @@
es_hosts: "{% set comma = joiner(',') %}{% for host in groups['search'] -%}{{ comma() }}{{ host }}{%- endfor %}"
roles:
- role: elasticsearch
+ - { role: metron_elasticsearch_templates, tags: ['load_es_templates'] }
tags:
- elasticsearch
@@ -96,10 +97,10 @@
- hosts: enrichment
become: true
- roles:
- - role: metron_streaming
tags:
- enrichment
+ roles:
+ - role: metron_streaming
#
# user interface
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/739e2eb5/metron-deployment/roles/elasticsearch/files/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/files/bro_index.template b/metron-deployment/roles/elasticsearch/files/bro_index.template
deleted file mode 100644
index 4da5110..0000000
--- a/metron-deployment/roles/elasticsearch/files/bro_index.template
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "template" : "bro_index*",
- "mappings" : {
- "bro_doc" : {
- "_timestamp" : { "enabled" : true },
- "properties": {
- "timestamp":{"type":"date","format":"epoch_millis"},
- "enrichments:geo:ip_dst_addr:location_point" : { "type" : "geo_point" }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/739e2eb5/metron-deployment/roles/elasticsearch/files/snort_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/files/snort_index.template b/metron-deployment/roles/elasticsearch/files/snort_index.template
deleted file mode 100644
index 01118ba..0000000
--- a/metron-deployment/roles/elasticsearch/files/snort_index.template
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "template": "snort_index*",
- "mappings": {
- "snort_doc": {
- "_timestamp": {
- "enabled": true
- },
- "properties": {
- "timestamp": {
- "type": "date",
- "format": "epoch_millis"
- },
- "enrichments:geo:ip_dst_addr:location_point": {
- "type": "geo_point"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/739e2eb5/metron-deployment/roles/elasticsearch/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/files/yaf_index.template b/metron-deployment/roles/elasticsearch/files/yaf_index.template
deleted file mode 100644
index 442321e..0000000
--- a/metron-deployment/roles/elasticsearch/files/yaf_index.template
+++ /dev/null
@@ -1,85 +0,0 @@
-{
- "template": "yaf_index*",
- "mappings": {
- "yaf_doc": {
- "_timestamp": {
- "enabled": true
- },
- "properties": {
- "timestamp": {
- "type": "date",
- "format": "epoch_millis"
- },
- "enrichments:geo:ip_dst_addr:location_point": {
- "type": "geo_point"
- },
- "end-time": {
- "type": "string"
- },
- "duration": {
- "type": "string"
- },
- "rtt": {
- "type": "string"
- },
- "proto": {
- "type": "string"
- },
- "sip": {
- "type": "string"
- },
- "sp": {
- "type": "string"
- },
- "dip": {
- "type": "string"
- },
- "dp": {
- "type": "string"
- },
- "iflags": {
- "type": "string"
- },
- "uflags": {
- "type": "string"
- },
- "riflags": {
- "type": "string"
- },
- "ruflags": {
- "type": "string"
- },
- "isn": {
- "type": "string"
- },
- "risn": {
- "type": "string"
- },
- "tag": {
- "type": "string"
- },
- "rtag": {
- "type": "string"
- },
- "pkt": {
- "type": "string"
- },
- "oct": {
- "type": "string"
- },
- "rpkt": {
- "type": "string"
- },
- "roct": {
- "type": "string"
- },
- "app": {
- "type": "string"
- },
- "end-reason": {
- "type": "string"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/739e2eb5/metron-deployment/roles/elasticsearch/tasks/create-indices.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/tasks/create-indices.yml b/metron-deployment/roles/elasticsearch/tasks/create-indices.yml
deleted file mode 100644
index 0f27351..0000000
--- a/metron-deployment/roles/elasticsearch/tasks/create-indices.yml
+++ /dev/null
@@ -1,51 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Start Elasticsearch
- service: name=elasticsearch state=started
-
-- name : Wait for Elasticsearch Host to Start
- wait_for:
- host: "{{ groups.search[0] }}"
- port: "{{ elasticsearch_web_port }}"
- delay: 10
- timeout: 300
-
-- name: Wait for Green Index Status
- uri:
- url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_cat/health"
- method: GET
- status_code: 200
- return_content: yes
- register: result
- until: result.content.find("green") != -1
- retries: 10
- delay: 60
-
-- name: Add Elasticsearch templates for topologies
- uri:
- url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_template/template_{{ item.sensor }}"
- method: PUT
- body: "{{ item.file }} | to_json "
- status_code: 200
- with_items:
- - { sensor: bro, file: "{{ lookup('file','bro_index.template') }}" }
- - { sensor: yaf, file: "{{ lookup('file','yaf_index.template') }}" }
- - { sensor: snort, file: "{{ lookup('file','snort_index.template') }}" }
-
-- name: Stop Elasticsearch
- service: name=elasticsearch state=stopped
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/739e2eb5/metron-deployment/roles/elasticsearch/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/tasks/main.yml b/metron-deployment/roles/elasticsearch/tasks/main.yml
index 71fb0b5..7478842 100644
--- a/metron-deployment/roles/elasticsearch/tasks/main.yml
+++ b/metron-deployment/roles/elasticsearch/tasks/main.yml
@@ -17,9 +17,6 @@
---
- include: elasticsearch.yml
-- include: create-indices.yml
- run_once: yes
-
- name: Create Logrotate Script for Elasticsearch
template:
src: "metron-elasticsearch-logrotate.yml"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/739e2eb5/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/bro_index.template b/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/bro_index.template
new file mode 100644
index 0000000..4da5110
--- /dev/null
+++ b/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/bro_index.template
@@ -0,0 +1,12 @@
+{
+ "template" : "bro_index*",
+ "mappings" : {
+ "bro_doc" : {
+ "_timestamp" : { "enabled" : true },
+ "properties": {
+ "timestamp":{"type":"date","format":"epoch_millis"},
+ "enrichments:geo:ip_dst_addr:location_point" : { "type" : "geo_point" }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/739e2eb5/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/snort_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/snort_index.template b/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/snort_index.template
new file mode 100644
index 0000000..01118ba
--- /dev/null
+++ b/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/snort_index.template
@@ -0,0 +1,19 @@
+{
+ "template": "snort_index*",
+ "mappings": {
+ "snort_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "enrichments:geo:ip_dst_addr:location_point": {
+ "type": "geo_point"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/739e2eb5/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/yaf_index.template b/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/yaf_index.template
new file mode 100644
index 0000000..442321e
--- /dev/null
+++ b/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/yaf_index.template
@@ -0,0 +1,85 @@
+{
+ "template": "yaf_index*",
+ "mappings": {
+ "yaf_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "enrichments:geo:ip_dst_addr:location_point": {
+ "type": "geo_point"
+ },
+ "end-time": {
+ "type": "string"
+ },
+ "duration": {
+ "type": "string"
+ },
+ "rtt": {
+ "type": "string"
+ },
+ "proto": {
+ "type": "string"
+ },
+ "sip": {
+ "type": "string"
+ },
+ "sp": {
+ "type": "string"
+ },
+ "dip": {
+ "type": "string"
+ },
+ "dp": {
+ "type": "string"
+ },
+ "iflags": {
+ "type": "string"
+ },
+ "uflags": {
+ "type": "string"
+ },
+ "riflags": {
+ "type": "string"
+ },
+ "ruflags": {
+ "type": "string"
+ },
+ "isn": {
+ "type": "string"
+ },
+ "risn": {
+ "type": "string"
+ },
+ "tag": {
+ "type": "string"
+ },
+ "rtag": {
+ "type": "string"
+ },
+ "pkt": {
+ "type": "string"
+ },
+ "oct": {
+ "type": "string"
+ },
+ "rpkt": {
+ "type": "string"
+ },
+ "roct": {
+ "type": "string"
+ },
+ "app": {
+ "type": "string"
+ },
+ "end-reason": {
+ "type": "string"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/739e2eb5/metron-deployment/roles/metron_elasticsearch_templates/tasks/load_templates.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_elasticsearch_templates/tasks/load_templates.yml b/metron-deployment/roles/metron_elasticsearch_templates/tasks/load_templates.yml
new file mode 100644
index 0000000..e4192c5
--- /dev/null
+++ b/metron-deployment/roles/metron_elasticsearch_templates/tasks/load_templates.yml
@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Start Elasticsearch
+ service: name=elasticsearch state=started
+
+- name : Wait for Elasticsearch Host to Start
+ wait_for:
+ host: "{{ groups.search[0] }}"
+ port: "{{ elasticsearch_web_port }}"
+ delay: 10
+ timeout: 300
+
+- name: Wait for Index to Become Available
+ uri:
+ url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_cat/health"
+ method: GET
+ status_code: 200
+ return_content: yes
+ register: result
+ until: result.content.find("green") != -1 or result.content.find("yellow") != -1
+ retries: 10
+ delay: 60
+
+- name: Add Elasticsearch templates for topologies
+ uri:
+ url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_template/{{ item | basename | replace('.template','') }}"
+ method: PUT
+ body: "{{ lookup('file',item) }}"
+ status_code: 200
+ with_fileglob: ./files/es_templates/*.template
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/739e2eb5/metron-deployment/roles/metron_elasticsearch_templates/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_elasticsearch_templates/tasks/main.yml b/metron-deployment/roles/metron_elasticsearch_templates/tasks/main.yml
new file mode 100644
index 0000000..16aa0be
--- /dev/null
+++ b/metron-deployment/roles/metron_elasticsearch_templates/tasks/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: load_templates.yml
+ run_once: true