You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/05/27 17:39:28 UTC
[3/3] incubator-metron git commit: METRON-152 Add support for ES
2.3.x and Kibana 4.5.0 (dlyle65535 via nickwallen) closes
apache/incubator-metron#135
METRON-152 Add support for ES 2.3.x and Kibana 4.5.0 (dlyle65535 via nickwallen) closes apache/incubator-metron#135
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/fb66ec90
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/fb66ec90
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/fb66ec90
Branch: refs/heads/master
Commit: fb66ec901cec11c3bb9fd159f68b6c098793f29e
Parents: 64b0f18
Author: dlyle65535 <dl...@gmail.com>
Authored: Fri May 27 13:38:22 2016 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Fri May 27 13:38:22 2016 -0400
----------------------------------------------------------------------
metron-deployment/amazon-ec2/conf/defaults.yml | 2 +-
.../inventory/full-dev-platform/group_vars/all | 2 +-
metron-deployment/playbooks/metron_install.yml | 2 +-
.../elasticsearch/files/bro_index.template | 12 +
.../elasticsearch/files/elasticsearch.repo | 9 +-
.../elasticsearch/files/snort_index.template | 19 +
.../elasticsearch/files/yaf_index.template | 7 +-
.../elasticsearch/tasks/create-indices.yml | 11 +-
.../roles/elasticsearch/tasks/elasticsearch.yml | 20 +-
.../roles/kibana/defaults/main.yml | 18 +
.../roles/kibana/tasks/dependencies.yml | 27 +
.../roles/kibana/tasks/install_kibana.yml | 45 ++
metron-deployment/roles/kibana/tasks/main.yml | 20 +
.../roles/kibana/templates/kibana.repo.template | 22 +
.../metron_streaming/files/yaf_index.template | 36 -
metron-deployment/roles/metron_ui/README.md | 14 -
.../roles/metron_ui/defaults/main.yml | 21 -
.../roles/metron_ui/tasks/copy-source.yml | 46 --
.../roles/metron_ui/tasks/dependencies.yml | 33 -
.../roles/metron_ui/tasks/main.yml | 20 -
.../roles/metron_ui/tasks/metron-ui.yml | 57 --
.../roles/monit/tasks/monit-definitions.yml | 4 +-
.../roles/monit/templates/monit/kibana.monit | 22 +
.../roles/monit/templates/monit/metron-ui.monit | 22 -
metron-platform/elasticsearch-shaded/pom.xml | 84 +++
metron-platform/metron-common/pom.xml | 17 +-
.../common/interfaces/FieldNameConverter.java | 24 +
metron-platform/metron-data-management/pom.xml | 5 +-
.../bulk/ElasticsearchDataPrunerRunner.java | 18 +-
.../ElasticsearchDataPrunerIntegrationTest.java | 153 -----
.../bulk/ElasticsearchDataPrunerTest.java | 672 ++++++++++++++++++-
metron-platform/metron-elasticsearch/pom.xml | 34 +-
.../writer/ElasticsearchFieldNameConverter.java | 32 +
.../writer/ElasticsearchWriter.java | 78 ++-
.../ElasticsearchEnrichmentIntegrationTest.java | 9 +
.../components/ElasticSearchComponent.java | 31 +-
.../ElasticsearchFieldNameConverterTest.java | 32 +
.../enrichment/adapters/geo/GeoAdapter.java | 2 +-
.../enrichment/adapters/geo/GeoAdapterTest.java | 2 +-
.../integration/EnrichmentIntegrationTest.java | 139 ++--
.../metron/integration/mock/MockGeoAdapter.java | 2 +-
.../main/sample/data/test/indexed/TestIndexed | 20 +-
.../src/main/sample/data/yaf/indexed/YafIndexed | 20 +-
metron-platform/metron-parsers/pom.xml | 28 +-
metron-platform/metron-pcap/pom.xml | 15 +
.../SolrEnrichmentIntegrationTest.java | 6 +
metron-platform/pom.xml | 7 +-
47 files changed, 1345 insertions(+), 576 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/amazon-ec2/conf/defaults.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/conf/defaults.yml b/metron-deployment/amazon-ec2/conf/defaults.yml
index a505588..0734ec5 100644
--- a/metron-deployment/amazon-ec2/conf/defaults.yml
+++ b/metron-deployment/amazon-ec2/conf/defaults.yml
@@ -20,7 +20,7 @@ services_to_start:
- mysql
- elasticsearch
- pcap-service
- - metron-ui
+ - kibana
- yaf
- snort
- snort-logs
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/inventory/full-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all b/metron-deployment/inventory/full-dev-platform/group_vars/all
index 2d4dc7e..0ffac8b 100644
--- a/metron-deployment/inventory/full-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/full-dev-platform/group_vars/all
@@ -20,7 +20,7 @@ services_to_start:
- mysql
- elasticsearch
- pcap-service
- - metron-ui
+ - kibana
- yaf
- snort
- snort-logs
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml
index 42bbeb1..69f35f5 100644
--- a/metron-deployment/playbooks/metron_install.yml
+++ b/metron-deployment/playbooks/metron_install.yml
@@ -106,7 +106,7 @@
- hosts: web
become: true
roles:
- - { role: metron_ui, when: install_elasticsearch | default(True) == True }
+ - { role: kibana, when: install_elasticsearch | default(True) == True }
tags:
- web
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/elasticsearch/files/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/files/bro_index.template b/metron-deployment/roles/elasticsearch/files/bro_index.template
new file mode 100644
index 0000000..4da5110
--- /dev/null
+++ b/metron-deployment/roles/elasticsearch/files/bro_index.template
@@ -0,0 +1,12 @@
+{
+ "template" : "bro_index*",
+ "mappings" : {
+ "bro_doc" : {
+ "_timestamp" : { "enabled" : true },
+ "properties": {
+ "timestamp":{"type":"date","format":"epoch_millis"},
+ "enrichments:geo:ip_dst_addr:location_point" : { "type" : "geo_point" }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/elasticsearch/files/elasticsearch.repo
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/files/elasticsearch.repo b/metron-deployment/roles/elasticsearch/files/elasticsearch.repo
index f033ced..51dba70 100644
--- a/metron-deployment/roles/elasticsearch/files/elasticsearch.repo
+++ b/metron-deployment/roles/elasticsearch/files/elasticsearch.repo
@@ -15,9 +15,10 @@
# limitations under the License.
#
-[elasticsearch-1.7]
-name=Elasticsearch repository for 1.7.x packages
-baseurl=http://packages.elastic.co/elasticsearch/1.7/centos
+[elasticsearch-2.x]
+name=Elasticsearch repository for 2.x packages
+baseurl=https://packages.elastic.co/elasticsearch/2.x/centos
gpgcheck=1
-gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
+gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/elasticsearch/files/snort_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/files/snort_index.template b/metron-deployment/roles/elasticsearch/files/snort_index.template
new file mode 100644
index 0000000..01118ba
--- /dev/null
+++ b/metron-deployment/roles/elasticsearch/files/snort_index.template
@@ -0,0 +1,19 @@
+{
+ "template": "snort_index*",
+ "mappings": {
+ "snort_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "enrichments:geo:ip_dst_addr:location_point": {
+ "type": "geo_point"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/elasticsearch/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/files/yaf_index.template b/metron-deployment/roles/elasticsearch/files/yaf_index.template
index 15ff6f8..442321e 100644
--- a/metron-deployment/roles/elasticsearch/files/yaf_index.template
+++ b/metron-deployment/roles/elasticsearch/files/yaf_index.template
@@ -2,12 +2,15 @@
"template": "yaf_index*",
"mappings": {
"yaf_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
"properties": {
"timestamp": {
"type": "date",
- "format": "dateOptionalTime"
+ "format": "epoch_millis"
},
- "location_point": {
+ "enrichments:geo:ip_dst_addr:location_point": {
"type": "geo_point"
},
"end-time": {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/elasticsearch/tasks/create-indices.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/tasks/create-indices.yml b/metron-deployment/roles/elasticsearch/tasks/create-indices.yml
index d91d220..0f27351 100644
--- a/metron-deployment/roles/elasticsearch/tasks/create-indices.yml
+++ b/metron-deployment/roles/elasticsearch/tasks/create-indices.yml
@@ -38,11 +38,14 @@
- name: Add Elasticsearch templates for topologies
uri:
- url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_template/template_yaf"
- method: POST
- body: "{{ lookup('file','yaf_index.template') }}"
+ url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_template/template_{{ item.sensor }}"
+ method: PUT
+ body: "{{ item.file }} | to_json "
status_code: 200
- body_format: json
+ with_items:
+ - { sensor: bro, file: "{{ lookup('file','bro_index.template') }}" }
+ - { sensor: yaf, file: "{{ lookup('file','yaf_index.template') }}" }
+ - { sensor: snort, file: "{{ lookup('file','snort_index.template') }}" }
- name: Stop Elasticsearch
service: name=elasticsearch state=stopped
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml b/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml
index 41b7f37..b5df7b9 100644
--- a/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml
+++ b/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml
@@ -46,16 +46,28 @@
with_items:
- '{{ elasticsearch_data_dir.split(",") }}'
+- name: Configure Elasticsearch - remove stale entries
+ lineinfile: >
+ dest=/etc/elasticsearch/elasticsearch.yml
+ regexp="{{ item }}"
+ state=absent
+ with_items:
+ - '^cluster\.name:'
+ - '^network\.host:'
+ - '^discovery\.zen\.ping\.unicast\.hosts:'
+ - '^path\.data'
+
- name: Configure Elasticsearch
lineinfile: >
dest=/etc/elasticsearch/elasticsearch.yml
regexp="{{ item.regexp }}"
+ insertafter="{{ item.regexp }}"
line="{{ item.line }}"
state=present
with_items:
- - { regexp: '#cluster\.name', line: 'cluster.name: metron' }
- - { regexp: '#network\.host:', line: 'network.host: _{{
+ - { regexp: '^# *cluster\.name:', line: 'cluster.name: metron' }
+ - { regexp: '^# *network\.host:', line: 'network.host: _{{
elasticsearch_network_interface }}:ipv4_' }
- - { regexp: '#discovery\.zen\.ping\.unicast\.hosts',
+ - { regexp: '^# *discovery\.zen\.ping\.unicast\.hosts:',
line: 'discovery.zen.ping.unicast.hosts: [ {{ es_hosts }} ]'}
- - { regexp: '#path\.data', line: 'path.data: {{ elasticsearch_data_dir }}' }
+ - { regexp: '^# *path\.data', line: 'path.data: {{ elasticsearch_data_dir }}' }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/kibana/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kibana/defaults/main.yml b/metron-deployment/roles/kibana/defaults/main.yml
new file mode 100644
index 0000000..073d4b8
--- /dev/null
+++ b/metron-deployment/roles/kibana/defaults/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+kibana_repo_url: http://packages.elastic.co/kibana/4.5/centos
+kibana_repo_key_url: http://packages.elastic.co/GPG-KEY-elasticsearch
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/kibana/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kibana/tasks/dependencies.yml b/metron-deployment/roles/kibana/tasks/dependencies.yml
new file mode 100644
index 0000000..7450665
--- /dev/null
+++ b/metron-deployment/roles/kibana/tasks/dependencies.yml
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Load Elastic Public Signing Key
+ rpm_key:
+ state: present
+ key: "{{ kibana_repo_key_url }}"
+
+- name: Create kibana.repo
+ template:
+ src: kibana.repo.template
+ dest: /etc/yum.repos.d/kibana.repo
+ mode: 0644
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/kibana/tasks/install_kibana.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kibana/tasks/install_kibana.yml b/metron-deployment/roles/kibana/tasks/install_kibana.yml
new file mode 100644
index 0000000..7df077e
--- /dev/null
+++ b/metron-deployment/roles/kibana/tasks/install_kibana.yml
@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install Kibana
+ yum:
+ name: kibana
+ state: installed
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+
+- name: Configure Kibana - remove stale entries
+ lineinfile: >
+ dest=/opt/kibana/config/kibana.yml
+ regexp="{{ item }}"
+ state=absent
+ with_items:
+ - '^server\.port:'
+ - '^elasticsearch\.url:'
+
+- name: Configure Kibana
+ lineinfile: >
+ dest=/opt/kibana/config/kibana.yml
+ regexp="{{ item.regexp }}"
+ insertafter="{{ item.regexp }}"
+ line="{{ item.line }}"
+ state=present
+ with_items:
+ - { regexp: '^# *server\.port:', line: 'server.port: 5000' }
+ - { regexp: '^# *elasticsearch\.url:', line: 'elasticsearch.url: http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}' }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/kibana/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kibana/tasks/main.yml b/metron-deployment/roles/kibana/tasks/main.yml
new file mode 100644
index 0000000..280223b
--- /dev/null
+++ b/metron-deployment/roles/kibana/tasks/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: dependencies.yml
+- include: install_kibana.yml
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/kibana/templates/kibana.repo.template
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kibana/templates/kibana.repo.template b/metron-deployment/roles/kibana/templates/kibana.repo.template
new file mode 100644
index 0000000..d5c096f
--- /dev/null
+++ b/metron-deployment/roles/kibana/templates/kibana.repo.template
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+[kibana]
+name=Kibana installation repository
+baseurl={{ kibana_repo_url }}
+gpgcheck=1
+gpgkey={{ kibana_repo_key_url }}
+enabled=1
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/metron_streaming/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/files/yaf_index.template b/metron-deployment/roles/metron_streaming/files/yaf_index.template
deleted file mode 100644
index c8c1702..0000000
--- a/metron-deployment/roles/metron_streaming/files/yaf_index.template
+++ /dev/null
@@ -1,36 +0,0 @@
-{
- "template" : "yaf_index*",
- "mappings" : {
- "yaf_doc" : {
- "properties": {
- "message": {
- "properties": {
- "@timestamp":{"type":"date","format":"dateOptionalTime"},
- "end-time":{"type":"string"},
- "duration":{"type":"string"},
- "rtt":{"type":"string"},
- "proto":{"type":"string"},
- "sip":{"type":"string"},
- "sp":{"type":"string"},
- "dip":{"type":"string"},
- "dp":{"type":"string"},
- "iflags":{"type":"string"},
- "uflags":{"type":"string"},
- "riflags":{"type":"string"},
- "ruflags":{"type":"string"},
- "isn":{"type":"string"},
- "risn":{"type":"string"},
- "tag":{"type":"string"},
- "rtag":{"type":"string"},
- "pkt":{"type":"string"},
- "oct":{"type":"string"},
- "rpkt":{"type":"string"},
- "roct":{"type":"string"},
- "app":{"type":"string"},
- "end-reason":{"type":"string"}
- }
- }
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/metron_ui/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_ui/README.md b/metron-deployment/roles/metron_ui/README.md
deleted file mode 100644
index 54eb6e4..0000000
--- a/metron-deployment/roles/metron_ui/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-Metron Deployment - Metron UI
-=============================
-
-This project deploys all of the necessary components to run the Metron UI. This is installed as a SysV service for management purposes.
-
-Getting Started
----------------
-
-```
-service pm2-init.sh start
-service pm2-init.sh stop
-service pm2-init.sh restart
-service pm2-init.sh status
-```
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/metron_ui/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_ui/defaults/main.yml b/metron-deployment/roles/metron_ui/defaults/main.yml
deleted file mode 100644
index 23aed40..0000000
--- a/metron-deployment/roles/metron_ui/defaults/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-metron_version: 0.1BETA
-metron_directory: /usr/metron/{{ metron_version }}
-metron_ui_directory: "{{ metron_directory }}/metron-ui"
-metron_temp_archive: /tmp/metron-ui.tar.gz
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/metron_ui/tasks/copy-source.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_ui/tasks/copy-source.yml b/metron-deployment/roles/metron_ui/tasks/copy-source.yml
deleted file mode 100644
index 703b7f9..0000000
--- a/metron-deployment/roles/metron_ui/tasks/copy-source.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Verify {{ metron_ui_directory }} exists
- file:
- path: "{{ metron_ui_directory }}"
- state: directory
- mode: 0755
- owner: root
- group: root
-
-- name: Archive metron-ui on localhost
- shell: tar --exclude='./node_modules' -czf {{ metron_temp_archive }} .
- args:
- chdir: "{{ playbook_dir }}/../../metron-ui"
- creates: "{{ metron_temp_archive }}"
- warn: false #Warns to use unarchive - unarchive does not archive
- become: false
- delegate_to: localhost
- run_once: true
-
-- name: Extract metron-ui tarball
- unarchive:
- src: "{{ metron_temp_archive }}"
- dest: "{{ metron_ui_directory }}"
- creates: "{{ metron_ui_directory}}/config"
-
-- name: Delete {{ metron_temp_archive }}
- local_action: file path="{{ metron_temp_archive }}" state=absent
- become: false
- run_once: true
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/metron_ui/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_ui/tasks/dependencies.yml b/metron-deployment/roles/metron_ui/tasks/dependencies.yml
deleted file mode 100644
index fd849f7..0000000
--- a/metron-deployment/roles/metron_ui/tasks/dependencies.yml
+++ /dev/null
@@ -1,33 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install yum repositories
- yum: name=epel-release update_cache=yes
-
-- name: Install Metron UI dependencies
- yum:
- pkg: "{{ item }}"
- state: installed
- with_items:
- - libpcap-devel
- - wireshark
- - nodejs
- - npm
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/metron_ui/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_ui/tasks/main.yml b/metron-deployment/roles/metron_ui/tasks/main.yml
deleted file mode 100644
index f7bb733..0000000
--- a/metron-deployment/roles/metron_ui/tasks/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: dependencies.yml
-- include: copy-source.yml
-- include: metron-ui.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/metron_ui/tasks/metron-ui.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_ui/tasks/metron-ui.yml b/metron-deployment/roles/metron_ui/tasks/metron-ui.yml
deleted file mode 100644
index 1e4d30a..0000000
--- a/metron-deployment/roles/metron_ui/tasks/metron-ui.yml
+++ /dev/null
@@ -1,57 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Configure Metron UI
- lineinfile:
- dest="{{ metron_ui_directory }}/config.json"
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- state=present
- with_items:
- - { regexp: '"elasticsearch":', line: '"elasticsearch": { "url": "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}" },' }
- - { regexp: '"pcap":', line: ' "pcap": { "url": "http://{{ groups.pcap_server[0] }}:{{ pcapservice_port }}/pcapGetter","mock": false }' }
-
-- name: Install Node dependencies
- npm:
- name: pm2
- path: "{{ metron_ui_directory }}"
- global: true
-
-- name: Install Metron UI
- npm:
- path: "{{ metron_ui_directory }}"
- production: no
-
-- name: Ensure Metron UI is stopped before installing service
- shell: pm2 stop all
- args:
- creates: /etc/init.d/pm2-init.sh
- register: pm2_result
- failed_when: pm2_result.rc != 0 and ("No process found" not in pm2_result.stderr)
-
-- name: Configure Metron UI as a service
- shell: "{{ item }}"
- args:
- creates: /etc/init.d/pm2-init.sh
- with_items:
- - "pm2 start {{ metron_ui_directory }}/lib/metron-ui.js --name metron"
- - pm2 save
- - pm2 startup centos
- - su -c 'chmod +x /etc/init.d/pm2-init.sh'
-
-- name: Stop Metron UI
- service: name=pm2-init.sh state=stopped
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/monit/tasks/monit-definitions.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/monit-definitions.yml b/metron-deployment/roles/monit/tasks/monit-definitions.yml
index 18ea0b0..3778e14 100644
--- a/metron-deployment/roles/monit/tasks/monit-definitions.yml
+++ b/metron-deployment/roles/monit/tasks/monit-definitions.yml
@@ -28,8 +28,8 @@
template: src=monit/enrichment-elasticsearch.monit dest={{ monit_config_home }}/enrichment-elasticsearch.monit
when: ("enrichment" in group_names) and (install_elasticsearch | default(True))
-- name: Create monit definition for metron-ui
- template: src=monit/metron-ui.monit dest={{ monit_config_home }}/metron-ui.monit
+- name: Create monit definition for kibana
+ template: src=monit/kibana.monit dest={{ monit_config_home }}/kibana.monit
when: ("web" in group_names) and (install_elasticsearch | default(True))
- name: Create monit definition for mysql
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/monit/templates/monit/kibana.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/kibana.monit b/metron-deployment/roles/monit/templates/monit/kibana.monit
new file mode 100644
index 0000000..41b4cb9
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/kibana.monit
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check process kibana matching "/opt/kibana/bin/../node/bin/node /opt/kibana/bin/../src/cli"
+ start program = "/etc/init.d/kibana start"
+ stop program = "/etc/init.d/kibana stop"
+ if does not exist then restart
+ group web
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-deployment/roles/monit/templates/monit/metron-ui.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/metron-ui.monit b/metron-deployment/roles/monit/templates/monit/metron-ui.monit
deleted file mode 100644
index 1b7e9cc..0000000
--- a/metron-deployment/roles/monit/templates/monit/metron-ui.monit
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-check process metron-ui matching "node /usr/metron/{{ metron_version }}/metron-ui/lib/metron-ui.js"
- start program = "/etc/init.d/pm2-init.sh start"
- stop program = "/etc/init.d/pm2-init.sh stop"
- if does not exist then restart
- group web
- group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-platform/elasticsearch-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml
new file mode 100644
index 0000000..2a85e7a
--- /dev/null
+++ b/metron-platform/elasticsearch-shaded/pom.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>metron-platform</artifactId>
+ <groupId>org.apache.metron</groupId>
+ <version>0.1BETA</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>elasticsearch-shaded</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>18.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${global_elasticsearch_version}</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${global_shade_version}</version>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>org.apache.metron.guava.elasticsearch-shaded</shadedPattern>
+ </relocation>
+ </relocations>
+ <artifactSet>
+ <excludes>
+ <exclude>storm:storm-core:*</exclude>
+ <exclude>storm:storm-lib:*</exclude>
+ <exclude>org.slf4j.impl*</exclude>
+ <exclude>org.slf4j:slf4j-log4j*</exclude>
+ </excludes>
+ </artifactSet>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resource>.yaml</resource>
+ </transformer>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass></mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index 9d439e0..172d387 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -151,11 +151,20 @@
</exclusions>
</dependency>
<dependency>
- <groupId>com.github.fge</groupId>
- <artifactId>json-schema-validator</artifactId>
- <version>${global_json_schema_validator_version}</version>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${global_jackson_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${global_jackson_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${global_jackson_version}</version>
</dependency>
-
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-platform/metron-common/src/main/java/org/apache/metron/common/interfaces/FieldNameConverter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/interfaces/FieldNameConverter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/interfaces/FieldNameConverter.java
new file mode 100644
index 0000000..92e7ec6
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/interfaces/FieldNameConverter.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.interfaces;
+
+public interface FieldNameConverter {
+
+ String convert(String originalField);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-platform/metron-data-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml
index 941c265..68ffc65 100644
--- a/metron-platform/metron-data-management/pom.xml
+++ b/metron-platform/metron-data-management/pom.xml
@@ -25,6 +25,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<httpcore.version>4.3.2</httpcore.version>
+ <lucene.test.version>5.5.0</lucene.test.version>
</properties>
<dependencies>
<dependency>
@@ -167,13 +168,13 @@
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-test-framework</artifactId>
- <version>4.10.4</version>
+ <version>${lucene.test.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
- <version>4.10.4</version>
+ <version>${lucene.test.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java
index f0a4d3b..d424004 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java
@@ -24,12 +24,14 @@ import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.metron.common.configuration.Configuration;
import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.Date;
@@ -91,11 +93,15 @@ public class ElasticsearchDataPrunerRunner {
configuration.update();
Map<String, Object> globalConfiguration = configuration.getGlobalConfig();
- ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder();
- builder.put("cluster.name", globalConfiguration.get("es.clustername"));
- builder.put("curatorFramework.transport.ping_timeout","500s");
- client = new TransportClient(builder.build())
- .addTransportAddress(new InetSocketTransportAddress(globalConfiguration.get("es.ip").toString(), Integer.parseInt(globalConfiguration.get("es.port").toString())));
+
+ Settings.Builder settingsBuilder = Settings.settingsBuilder();
+ settingsBuilder.put("cluster.name", globalConfiguration.get("es.clustername"));
+ settingsBuilder.put("curatorFramework.transport.ping_timeout","500s");
+ Settings settings = settingsBuilder.build();
+ client = TransportClient.builder().settings(settings).build()
+ .addTransportAddress(
+ new InetSocketTransportAddress(InetAddress.getByName(globalConfiguration.get("es.ip").toString()), Integer.parseInt(globalConfiguration.get("es.port").toString()) )
+ );
DataPruner pruner = new ElasticsearchDataPruner(startDate, numDays, configuration, client, indexPrefix);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerIntegrationTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerIntegrationTest.java
deleted file mode 100644
index 08f95b9..0000000
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerIntegrationTest.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.dataloads.bulk;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.metron.TestConstants;
-import org.apache.metron.common.configuration.Configuration;
-import org.elasticsearch.common.settings.ImmutableSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.indices.IndexMissingException;
-import org.elasticsearch.test.ElasticsearchIntegrationTest;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.TimeZone;
-import java.util.concurrent.TimeUnit;
-
-@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.SUITE, numDataNodes = 1, numClientNodes = 0)
-public class ElasticsearchDataPrunerIntegrationTest extends ElasticsearchIntegrationTest {
-
- private static File dataPath = new File("./target/elasticsearch-test");
- private Date testingDate;
- private Date yesterday = new Date();
- private DateFormat dateFormat = new SimpleDateFormat("yyyy.MM.dd.HH");
- private Configuration configuration;
-
- @BeforeClass
- public static void setupClass() throws Exception {
-
- if (dataPath.isDirectory()) {
- FileUtils.deleteDirectory(dataPath);
- }
-
- if (!dataPath.mkdirs()) {
- throw new RuntimeException("Couldn't create dataPath at: " + dataPath.getAbsolutePath());
- }
-
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
-
- if (dataPath.isDirectory()) {
- FileUtils.deleteDirectory(dataPath);
- }
-
- }
-
- @Before
- public void setUp() throws Exception {
-
- super.setUp();
- ensureGreen();
-
- TimeZone timeZone = TimeZone.getTimeZone("UTC");
- Calendar calendar = Calendar.getInstance(timeZone);
- calendar.set(Calendar.HOUR_OF_DAY,0);
- calendar.set(Calendar.MINUTE,0);
- calendar.set(Calendar.SECOND,0);
- testingDate = calendar.getTime();
- yesterday.setTime(testingDate.getTime() - TimeUnit.DAYS.toMillis(1));
- dateFormat.setTimeZone(timeZone);
-
- File resourceFile = new File(TestConstants.SAMPLE_CONFIG_PATH);
- Path resourcePath = Paths.get(resourceFile.getCanonicalPath());
-
- configuration = new Configuration(resourcePath);
- }
-
- @Test(expected = IndexMissingException.class)
- public void testWillThrowOnMissingIndex() throws Exception {
-
- ElasticsearchDataPruner pruner = new ElasticsearchDataPruner(yesterday, 30, configuration,client(), "*");
- pruner.deleteIndex(admin(), "baz");
-
- }
-
- @Test
- public void testDeletesCorrectIndexes() throws Exception {
-
- Integer numDays = 5;
-
- Date createStartDate = new Date();
-
- createStartDate.setTime(yesterday.getTime() - TimeUnit.DAYS.toMillis(numDays - 1));
-
- ElasticsearchDataPruner pruner = new ElasticsearchDataPruner(yesterday, 30, configuration,client(), "*");
- String indexesToDelete = "sensor_index_" + new SimpleDateFormat("yyyy.MM.dd").format(createStartDate) + ".*";
- Boolean deleted = pruner.deleteIndex(admin(), indexesToDelete);
-
- assertTrue("Index deletion should be acknowledged", deleted);
-
- }
-
- @Test
- public void testHandlesNoIndicesToDelete() throws Exception {
-
- ElasticsearchDataPruner pruner = new ElasticsearchDataPruner(yesterday, 1, configuration, client(), "sensor_index_");
- Long deleteCount = pruner.prune();
- assertEquals("Should have pruned 0 indices", 0L, deleteCount.longValue());
-
-
- }
-
- @Override
- protected Settings nodeSettings(int nodeOrdinal) {
-
- return ImmutableSettings.settingsBuilder()
- .put("node.data", true)
- .put("gateway.type", "none")
- .put("path.data", dataPath.getPath() + "/data")
- .put("path.work", dataPath.getPath() + "/work")
- .put("path.logs", dataPath.getPath() + "/logs")
- .put("cluster.routing.schedule", "50ms")
- .put("node.local", true).build();
-
- }
-
- public Settings indexSettings() {
-
- return ImmutableSettings.settingsBuilder()
- .put("index.store.type", "memory")
- .put("index.store.fs.memory.enabled", "true")
- .put("index.number_of_shards", 1)
- .put("index.number_of_replicas", 0).build();
-
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerTest.java
index 0cc5d28..df485f0 100644
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerTest.java
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerTest.java
@@ -17,28 +17,119 @@
*/
package org.apache.metron.dataloads.bulk;
+import com.carrotsearch.hppc.ObjectObjectHashMap;
import org.apache.commons.collections.IteratorUtils;
import org.apache.metron.TestConstants;
import org.apache.metron.common.configuration.Configuration;
import org.easymock.EasyMock;
-import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.*;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
+import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistRequestBuilder;
+import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
+import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
+import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder;
+import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
+import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
+import org.elasticsearch.action.admin.indices.close.CloseIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
-import org.elasticsearch.client.*;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
+import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest;
+import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequestBuilder;
+import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;
+import org.elasticsearch.action.admin.indices.flush.*;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.*;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
+import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest;
+import org.elasticsearch.action.admin.indices.recovery.RecoveryRequestBuilder;
+import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder;
+import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
+import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse;
+import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest;
+import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequestBuilder;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequestBuilder;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsResponse;
+import org.elasticsearch.action.admin.indices.shards.IndicesShardStoreRequestBuilder;
+import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest;
+import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder;
+import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse;
+import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
+import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequestBuilder;
+import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
+import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusRequest;
+import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusRequestBuilder;
+import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusResponse;
+import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
+import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequestBuilder;
+import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeResponse;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
+import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerRequest;
+import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerRequestBuilder;
+import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerResponse;
+import org.elasticsearch.action.admin.indices.warmer.get.GetWarmersRequest;
+import org.elasticsearch.action.admin.indices.warmer.get.GetWarmersRequestBuilder;
+import org.elasticsearch.action.admin.indices.warmer.get.GetWarmersResponse;
+import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerRequest;
+import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerRequestBuilder;
+import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerResponse;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.ClusterAdminClient;
+import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.hppc.ObjectObjectOpenHashMap;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.indices.IndexMissingException;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.threadpool.ThreadPool;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.mockito.Matchers;
import org.powermock.api.easymock.PowerMock;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -57,7 +148,6 @@ import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.powermock.api.easymock.PowerMock.replayAll;
@@ -73,7 +163,7 @@ public class ElasticsearchDataPrunerTest {
private Client indexClient = mock(Client.class);
private AdminClient adminClient = mock(AdminClient.class);
- private IndicesAdminClient indicesAdminClient = mock(FilterClient.IndicesAdmin.class);
+ private IndicesAdminClient indicesAdminClient = new TestIndicesAdminClient();
private DeleteIndexRequestBuilder deleteIndexRequestBuilder = mock(DeleteIndexRequestBuilder.class);
private DeleteIndexRequest deleteIndexRequest = mock(DeleteIndexRequest.class);
private ActionFuture<DeleteIndexResponse> deleteIndexAction = mock(ActionFuture.class);
@@ -98,8 +188,6 @@ public class ElasticsearchDataPrunerTest {
when(indexClient.admin()).thenReturn(adminClient);
when(adminClient.indices()).thenReturn(indicesAdminClient);
- when(indicesAdminClient.prepareDelete(Matchers.<String>anyVararg())).thenReturn(deleteIndexRequestBuilder);
- when(indicesAdminClient.delete((DeleteIndexRequest) any())).thenReturn(deleteIndexAction);
when(deleteIndexRequestBuilder.request()).thenReturn(deleteIndexRequest);
when(deleteIndexAction.actionGet()).thenReturn(deleteIndexResponse);
@@ -116,12 +204,13 @@ public class ElasticsearchDataPrunerTest {
}
- @Test(expected = IndexMissingException.class)
+ @Test(expected = IndexNotFoundException.class)
public void testWillThrowOnMissingIndex() throws Exception {
- when(indicesAdminClient.delete((DeleteIndexRequest) any())).thenThrow(new IndexMissingException(new Index("Test Exception")));
+ ((TestIndicesAdminClient)indicesAdminClient).throwMissingIndex = true;
ElasticsearchDataPruner pruner = new ElasticsearchDataPruner(testDate, 30, configuration, indexClient,"*");
pruner.deleteIndex(adminClient, "baz");
+ ((TestIndicesAdminClient)indicesAdminClient).throwMissingIndex = false;
}
@@ -133,7 +222,7 @@ public class ElasticsearchDataPrunerTest {
ClusterStateRequestBuilder clusterStateRequestBuilder = mock(ClusterStateRequestBuilder.class);
ClusterStateResponse clusterStateResponse = mock(ClusterStateResponse.class);
ClusterState clusterState = mock(ClusterState.class);
- ObjectObjectOpenHashMap<String, IndexMetaData> clusterIndexes = new ObjectObjectOpenHashMap();
+ ObjectObjectHashMap<String, IndexMetaData> clusterIndexes = new ObjectObjectHashMap();
MetaData clusterMetadata = mock(MetaData.class);
when(adminClient.cluster()).thenReturn(clusterAdminClient);
when(clusterAdminClient.prepareState()).thenReturn(clusterStateRequestBuilder);
@@ -172,7 +261,7 @@ public class ElasticsearchDataPrunerTest {
@Test
public void testFilter() throws Exception {
- ObjectObjectOpenHashMap<String, IndexMetaData> indexNames = new ObjectObjectOpenHashMap();
+ ObjectObjectHashMap<String, IndexMetaData> indexNames = new ObjectObjectHashMap();
SimpleDateFormat dateChecker = new SimpleDateFormat("yyyyMMdd");
int numDays = 5;
String[] expectedIndices = new String[24];
@@ -208,4 +297,559 @@ public class ElasticsearchDataPrunerTest {
}
+ class TestIndicesAdminClient implements IndicesAdminClient {
+
+ public boolean throwMissingIndex = false;
+
+ @Override
+ public ActionFuture<DeleteIndexResponse> delete(DeleteIndexRequest request) {
+
+ if(throwMissingIndex){
+
+ throw new IndexNotFoundException("TEST EXCEPTION!");
+
+ }
+
+ return deleteIndexAction;
+
+ }
+
+
+ @Override
+ public ActionFuture<IndicesExistsResponse> exists(IndicesExistsRequest request) {
+ return null;
+ }
+
+ @Override
+ public void exists(IndicesExistsRequest request, ActionListener<IndicesExistsResponse> listener) {
+
+ }
+
+ @Override
+ public IndicesExistsRequestBuilder prepareExists(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<TypesExistsResponse> typesExists(TypesExistsRequest request) {
+ return null;
+ }
+
+ @Override
+ public void typesExists(TypesExistsRequest request, ActionListener<TypesExistsResponse> listener) {
+
+ }
+
+ @Override
+ public TypesExistsRequestBuilder prepareTypesExists(String... index) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<IndicesStatsResponse> stats(IndicesStatsRequest request) {
+ return null;
+ }
+
+ @Override
+ public void stats(IndicesStatsRequest request, ActionListener<IndicesStatsResponse> listener) {
+
+ }
+
+ @Override
+ public IndicesStatsRequestBuilder prepareStats(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<RecoveryResponse> recoveries(RecoveryRequest request) {
+ return null;
+ }
+
+ @Override
+ public void recoveries(RecoveryRequest request, ActionListener<RecoveryResponse> listener) {
+
+ }
+
+ @Override
+ public RecoveryRequestBuilder prepareRecoveries(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<IndicesSegmentResponse> segments(IndicesSegmentsRequest request) {
+ return null;
+ }
+
+ @Override
+ public void segments(IndicesSegmentsRequest request, ActionListener<IndicesSegmentResponse> listener) {
+
+ }
+
+ @Override
+ public IndicesSegmentsRequestBuilder prepareSegments(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<IndicesShardStoresResponse> shardStores(IndicesShardStoresRequest request) {
+ return null;
+ }
+
+ @Override
+ public void shardStores(IndicesShardStoresRequest request, ActionListener<IndicesShardStoresResponse> listener) {
+
+ }
+
+ @Override
+ public IndicesShardStoreRequestBuilder prepareShardStores(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<CreateIndexResponse> create(CreateIndexRequest request) {
+ return null;
+ }
+
+ @Override
+ public void create(CreateIndexRequest request, ActionListener<CreateIndexResponse> listener) {
+
+ }
+
+ @Override
+ public CreateIndexRequestBuilder prepareCreate(String index) {
+ return null;
+ }
+
+
+ @Override
+ public void delete(DeleteIndexRequest request, ActionListener<DeleteIndexResponse> listener) {
+
+ }
+
+ @Override
+ public DeleteIndexRequestBuilder prepareDelete(String... indices) {
+ return deleteIndexRequestBuilder;
+ }
+
+ @Override
+ public ActionFuture<CloseIndexResponse> close(CloseIndexRequest request) {
+ return null;
+ }
+
+ @Override
+ public void close(CloseIndexRequest request, ActionListener<CloseIndexResponse> listener) {
+
+ }
+
+ @Override
+ public CloseIndexRequestBuilder prepareClose(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<OpenIndexResponse> open(OpenIndexRequest request) {
+ return null;
+ }
+
+ @Override
+ public void open(OpenIndexRequest request, ActionListener<OpenIndexResponse> listener) {
+
+ }
+
+ @Override
+ public OpenIndexRequestBuilder prepareOpen(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<RefreshResponse> refresh(RefreshRequest request) {
+ return null;
+ }
+
+ @Override
+ public void refresh(RefreshRequest request, ActionListener<RefreshResponse> listener) {
+
+ }
+
+ @Override
+ public RefreshRequestBuilder prepareRefresh(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<FlushResponse> flush(FlushRequest request) {
+ return null;
+ }
+
+ @Override
+ public void flush(FlushRequest request, ActionListener<FlushResponse> listener) {
+
+ }
+
+ @Override
+ public FlushRequestBuilder prepareFlush(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<SyncedFlushResponse> syncedFlush(SyncedFlushRequest request) {
+ return null;
+ }
+
+ @Override
+ public void syncedFlush(SyncedFlushRequest request, ActionListener<SyncedFlushResponse> listener) {
+
+ }
+
+ @Override
+ public SyncedFlushRequestBuilder prepareSyncedFlush(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<ForceMergeResponse> forceMerge(ForceMergeRequest request) {
+ return null;
+ }
+
+ @Override
+ public void forceMerge(ForceMergeRequest request, ActionListener<ForceMergeResponse> listener) {
+
+ }
+
+ @Override
+ public ForceMergeRequestBuilder prepareForceMerge(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<UpgradeResponse> upgrade(UpgradeRequest request) {
+ return null;
+ }
+
+ @Override
+ public void upgrade(UpgradeRequest request, ActionListener<UpgradeResponse> listener) {
+
+ }
+
+ @Override
+ public UpgradeStatusRequestBuilder prepareUpgradeStatus(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<UpgradeStatusResponse> upgradeStatus(UpgradeStatusRequest request) {
+ return null;
+ }
+
+ @Override
+ public void upgradeStatus(UpgradeStatusRequest request, ActionListener<UpgradeStatusResponse> listener) {
+
+ }
+
+ @Override
+ public UpgradeRequestBuilder prepareUpgrade(String... indices) {
+ return null;
+ }
+
+ @Override
+ public void getMappings(GetMappingsRequest request, ActionListener<GetMappingsResponse> listener) {
+
+ }
+
+ @Override
+ public ActionFuture<GetMappingsResponse> getMappings(GetMappingsRequest request) {
+ return null;
+ }
+
+ @Override
+ public GetMappingsRequestBuilder prepareGetMappings(String... indices) {
+ return null;
+ }
+
+ @Override
+ public void getFieldMappings(GetFieldMappingsRequest request, ActionListener<GetFieldMappingsResponse> listener) {
+
+ }
+
+ @Override
+ public GetFieldMappingsRequestBuilder prepareGetFieldMappings(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<GetFieldMappingsResponse> getFieldMappings(GetFieldMappingsRequest request) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<PutMappingResponse> putMapping(PutMappingRequest request) {
+ return null;
+ }
+
+ @Override
+ public void putMapping(PutMappingRequest request, ActionListener<PutMappingResponse> listener) {
+
+ }
+
+ @Override
+ public PutMappingRequestBuilder preparePutMapping(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<IndicesAliasesResponse> aliases(IndicesAliasesRequest request) {
+ return null;
+ }
+
+ @Override
+ public void aliases(IndicesAliasesRequest request, ActionListener<IndicesAliasesResponse> listener) {
+
+ }
+
+ @Override
+ public IndicesAliasesRequestBuilder prepareAliases() {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<GetAliasesResponse> getAliases(GetAliasesRequest request) {
+ return null;
+ }
+
+ @Override
+ public void getAliases(GetAliasesRequest request, ActionListener<GetAliasesResponse> listener) {
+
+ }
+
+ @Override
+ public GetAliasesRequestBuilder prepareGetAliases(String... aliases) {
+ return null;
+ }
+
+ @Override
+ public AliasesExistRequestBuilder prepareAliasesExist(String... aliases) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<AliasesExistResponse> aliasesExist(GetAliasesRequest request) {
+ return null;
+ }
+
+ @Override
+ public void aliasesExist(GetAliasesRequest request, ActionListener<AliasesExistResponse> listener) {
+
+ }
+
+ @Override
+ public ActionFuture<GetIndexResponse> getIndex(GetIndexRequest request) {
+ return null;
+ }
+
+ @Override
+ public void getIndex(GetIndexRequest request, ActionListener<GetIndexResponse> listener) {
+
+ }
+
+ @Override
+ public GetIndexRequestBuilder prepareGetIndex() {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<ClearIndicesCacheResponse> clearCache(ClearIndicesCacheRequest request) {
+ return null;
+ }
+
+ @Override
+ public void clearCache(ClearIndicesCacheRequest request, ActionListener<ClearIndicesCacheResponse> listener) {
+
+ }
+
+ @Override
+ public ClearIndicesCacheRequestBuilder prepareClearCache(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<UpdateSettingsResponse> updateSettings(UpdateSettingsRequest request) {
+ return null;
+ }
+
+ @Override
+ public void updateSettings(UpdateSettingsRequest request, ActionListener<UpdateSettingsResponse> listener) {
+
+ }
+
+ @Override
+ public UpdateSettingsRequestBuilder prepareUpdateSettings(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<AnalyzeResponse> analyze(AnalyzeRequest request) {
+ return null;
+ }
+
+ @Override
+ public void analyze(AnalyzeRequest request, ActionListener<AnalyzeResponse> listener) {
+
+ }
+
+ @Override
+ public AnalyzeRequestBuilder prepareAnalyze(@Nullable String index, String text) {
+ return null;
+ }
+
+ @Override
+ public AnalyzeRequestBuilder prepareAnalyze(String text) {
+ return null;
+ }
+
+ @Override
+ public AnalyzeRequestBuilder prepareAnalyze() {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<PutIndexTemplateResponse> putTemplate(PutIndexTemplateRequest request) {
+ return null;
+ }
+
+ @Override
+ public void putTemplate(PutIndexTemplateRequest request, ActionListener<PutIndexTemplateResponse> listener) {
+
+ }
+
+ @Override
+ public PutIndexTemplateRequestBuilder preparePutTemplate(String name) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<DeleteIndexTemplateResponse> deleteTemplate(DeleteIndexTemplateRequest request) {
+ return null;
+ }
+
+ @Override
+ public void deleteTemplate(DeleteIndexTemplateRequest request, ActionListener<DeleteIndexTemplateResponse> listener) {
+
+ }
+
+ @Override
+ public DeleteIndexTemplateRequestBuilder prepareDeleteTemplate(String name) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<GetIndexTemplatesResponse> getTemplates(GetIndexTemplatesRequest request) {
+ return null;
+ }
+
+ @Override
+ public void getTemplates(GetIndexTemplatesRequest request, ActionListener<GetIndexTemplatesResponse> listener) {
+
+ }
+
+ @Override
+ public GetIndexTemplatesRequestBuilder prepareGetTemplates(String... name) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<ValidateQueryResponse> validateQuery(ValidateQueryRequest request) {
+ return null;
+ }
+
+ @Override
+ public void validateQuery(ValidateQueryRequest request, ActionListener<ValidateQueryResponse> listener) {
+
+ }
+
+ @Override
+ public ValidateQueryRequestBuilder prepareValidateQuery(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<PutWarmerResponse> putWarmer(PutWarmerRequest request) {
+ return null;
+ }
+
+ @Override
+ public void putWarmer(PutWarmerRequest request, ActionListener<PutWarmerResponse> listener) {
+
+ }
+
+ @Override
+ public PutWarmerRequestBuilder preparePutWarmer(String name) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<DeleteWarmerResponse> deleteWarmer(DeleteWarmerRequest request) {
+ return null;
+ }
+
+ @Override
+ public void deleteWarmer(DeleteWarmerRequest request, ActionListener<DeleteWarmerResponse> listener) {
+
+ }
+
+ @Override
+ public DeleteWarmerRequestBuilder prepareDeleteWarmer() {
+ return null;
+ }
+
+ @Override
+ public void getWarmers(GetWarmersRequest request, ActionListener<GetWarmersResponse> listener) {
+
+ }
+
+ @Override
+ public ActionFuture<GetWarmersResponse> getWarmers(GetWarmersRequest request) {
+ return null;
+ }
+
+ @Override
+ public GetWarmersRequestBuilder prepareGetWarmers(String... indices) {
+ return null;
+ }
+
+ @Override
+ public void getSettings(GetSettingsRequest request, ActionListener<GetSettingsResponse> listener) {
+
+ }
+
+ @Override
+ public ActionFuture<GetSettingsResponse> getSettings(GetSettingsRequest request) {
+ return null;
+ }
+
+ @Override
+ public GetSettingsRequestBuilder prepareGetSettings(String... indices) {
+ return null;
+ }
+
+ @Override
+ public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
+ return null;
+ }
+
+ @Override
+ public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
+
+ }
+
+ @Override
+ public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(Action<Request, Response, RequestBuilder> action) {
+ return null;
+ }
+
+ @Override
+ public ThreadPool threadPool() {
+ return null;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-platform/metron-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index 90b091a..fb6f929 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -33,13 +33,13 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment</artifactId>
+ <artifactId>elasticsearch-shaded</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>${global_elasticsearch_version}</version>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-enrichment</artifactId>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
@@ -106,6 +106,21 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>3.10.5.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.2.3</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>2.2.3</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.metron</groupId>
<artifactId>metron-integration-test</artifactId>
<version>${project.parent.version}</version>
@@ -117,7 +132,6 @@
<version>${global_mockito_version}</version>
<scope>test</scope>
</dependency>
-
</dependencies>
<reporting>
<plugins>
@@ -206,6 +220,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>org.apache.metron.guava.metron-elasticsearch</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml.jackson.core</pattern>
+ <shadedPattern>com.fasterxml.jackson.core.metron.elasticsearch</shadedPattern>
+ </relocation>
+ </relocations>
<artifactSet>
<excludes>
<exclude>storm:storm-core:*</exclude>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fb66ec90/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchFieldNameConverter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchFieldNameConverter.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchFieldNameConverter.java
new file mode 100644
index 0000000..57e07ea
--- /dev/null
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchFieldNameConverter.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.elasticsearch.writer;
+
+import org.apache.metron.common.interfaces.FieldNameConverter;
+import java.io.Serializable;
+
+public class ElasticsearchFieldNameConverter implements FieldNameConverter, Serializable {
+
+ private static final long serialVersionUID = -3126840090749760299L;
+
+ @Override
+ public String convert(String originalField) {
+ return originalField.replace(".",":");
+ }
+
+}