You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ot...@apache.org on 2018/10/24 15:03:50 UTC
[01/51] [abbrv] metron git commit: Merge remote-tracking branch
'apache/master' into feature/METRON-1699-create-batch-profiler
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1090-stellar-assignment 9ce4ba5a9 -> 0e037edad
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --cc metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 38b3770,fca0949..4bcef33
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@@ -588,10 -546,8 +588,12 @@@ chkconfig --del metron-management-u
chkconfig --del metron-alerts-ui
%changelog
- * Tue Aug 21 2018 Apache Metron <de...@metron.apache.org> - 0.5.1
-* Thu Aug 30 2018 Apache Metron <de...@metron.apache.org> - 0.5.1
++* Thu Aug 30 2018 Apache Metron <de...@metron.apache.org> - 0.6.1
+ - Update compiled css file name for Alerts UI
++* Tue Aug 21 2018 Apache Metron <de...@metron.apache.org> - 0.6.1
+- Add Profiler for REPL
+* Tue Aug 14 2018 Apache Metron <de...@metron.apache.org> - 0.5.1
+- Add Profiler for Spark
* Thu Feb 1 2018 Apache Metron <de...@metron.apache.org> - 0.4.3
- Add Solr install script to Solr RPM
* Tue Sep 25 2017 Apache Metron <de...@metron.apache.org> - 0.4.2
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-deployment/packaging/docker/rpm-docker/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/pom.xml
----------------------------------------------------------------------
[17/51] [abbrv] metron git commit: METRON-1758 Add support for
Ansible 2.6 in dev (JonZeolla via jonzeolla) closes apache/metron#1179
Posted by ot...@apache.org.
METRON-1758 Add support for Ansible 2.6 in dev (JonZeolla via jonzeolla) closes apache/metron#1179
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1723a0ea
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1723a0ea
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1723a0ea
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 1723a0eae57adcfc4a2efe2422f52fbed4c237e7
Parents: fbabfca
Author: JonZeolla <ze...@gmail.com>
Authored: Mon Oct 1 08:41:25 2018 -0400
Committer: jonzeolla <jo...@apache.org>
Committed: Mon Oct 1 08:41:25 2018 -0400
----------------------------------------------------------------------
metron-deployment/amazon-ec2/README.md | 2 +-
metron-deployment/amazon-ec2/ansible.cfg | 1 +
metron-deployment/amazon-ec2/playbook.yml | 4 --
metron-deployment/ansible.cfg | 23 ----------
.../0_minimum_ansible_version.py | 45 ++++++++++++++++++++
.../ansible/playbooks/metron_full_install.yml | 16 +++----
.../ambari_common/tasks/ambari-repo-ubuntu.yml | 3 +-
.../ansible/roles/ambari_common/tasks/main.yml | 13 +++---
.../roles/ambari_common/tasks/nodejs.yml | 6 ++-
.../ansible/roles/enable-swap/tasks/main.yml | 3 +-
.../ansible/roles/java_jdk/tasks/main.yml | 6 +--
.../ansible/roles/metron-builder/tasks/main.yml | 6 +--
.../roles/metron-packages/tasks/main.yml | 12 ++----
metron-deployment/development/centos6/README.md | 5 +--
.../development/centos6/Vagrantfile | 2 +-
.../development/centos6/ansible.cfg | 2 +-
.../development/centos6/ansible/playbook.yml | 2 +-
.../development/fastcapa/centos-7.1/ansible.cfg | 2 +-
.../development/fastcapa/centos-7.4/ansible.cfg | 2 +-
.../development/ubuntu14/README.md | 5 +--
.../development/ubuntu14/Vagrantfile | 2 +-
.../development/ubuntu14/ansible.cfg | 2 +-
.../development/ubuntu14/ansible/playbook.yml | 2 +-
23 files changed, 84 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/amazon-ec2/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/README.md b/metron-deployment/amazon-ec2/README.md
index b2efc9e..ed1c73b 100644
--- a/metron-deployment/amazon-ec2/README.md
+++ b/metron-deployment/amazon-ec2/README.md
@@ -38,7 +38,7 @@ Getting Started
The host used to deploy Apache Metron will need the following software tools installed. The following versions are known to work as of the time of this writing, but by no means are these the only working versions.
- - Ansible 2.0.0.2, 2.2.2.0, or 2.5.0
+ - Ansible 2.4.0+
- Python 2.7.11
- Maven 3.3.9
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/amazon-ec2/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/ansible.cfg b/metron-deployment/amazon-ec2/ansible.cfg
index 0fe3343..4291cc8 100644
--- a/metron-deployment/amazon-ec2/ansible.cfg
+++ b/metron-deployment/amazon-ec2/ansible.cfg
@@ -22,6 +22,7 @@ pipelining = True
remote_user = centos
forks = 20
log_path = ./ansible.log
+callback_plugins = ../ansible/callback_plugins
# fix for "ssh throws 'unix domain socket too long' " problem
#[ssh_connection]
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/amazon-ec2/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/playbook.yml b/metron-deployment/amazon-ec2/playbook.yml
index cb1e326..4e8880a 100644
--- a/metron-deployment/amazon-ec2/playbook.yml
+++ b/metron-deployment/amazon-ec2/playbook.yml
@@ -21,10 +21,6 @@
- hosts: localhost
vars_files:
- conf/defaults.yml
- pre_tasks:
- - name: Verify Ansible Version
- fail: msg="Metron Requires Ansible 2.0.0.2, 2.2.2.0 or 2.5.0, current version is {{ ansible_version }}"
- when: "ansible_version.full | version_compare('2.2.2.0', '!=') and ansible_version.full | version_compare('2.0.0.2', '!=') and ansible_version.full | version_compare('2.5.0', '!=')"
tasks:
- include: tasks/create-keypair.yml
- include: tasks/create-vpc.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible.cfg b/metron-deployment/ansible.cfg
deleted file mode 100644
index 9b3916b..0000000
--- a/metron-deployment/ansible.cfg
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[defaults]
-host_key_checking = false
-library = extra_modules
-roles_path = ./roles
-
-[ssh_connection]
-control_path = %(directory)s/%%h-%%p-%%r
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/ansible/callback_plugins/0_minimum_ansible_version.py
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/callback_plugins/0_minimum_ansible_version.py b/metron-deployment/ansible/callback_plugins/0_minimum_ansible_version.py
new file mode 100644
index 0000000..596c132
--- /dev/null
+++ b/metron-deployment/ansible/callback_plugins/0_minimum_ansible_version.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+from ansible import __version__
+
+from ansible.plugins.callback import CallbackBase
+from ansible.utils.display import Display
+
+def display(*args, **kwargs):
+ display_instance = Display()
+ display_instance.display(*args, **kwargs)
+
+MINIMUM_ANSIBLE_VERSION = '2.4.0'
+
+def version_requirement(version):
+ return version >= MINIMUM_ANSIBLE_VERSION
+
+class CallbackModule(CallbackBase):
+ """
+ This enforces a minimum version of ansible
+ """
+
+ CALLBACK_NAME = 'minimum_ansible_version'
+
+ def __init__(self):
+ super(CallbackModule, self).__init__()
+ if not version_requirement(__version__):
+ display('Metron requires Ansible %s or newer, current version is %s' % (MINIMUM_ANSIBLE_VERSION, __version__), color='red')
+ sys.exit(1)
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/ansible/playbooks/metron_full_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/playbooks/metron_full_install.yml b/metron-deployment/ansible/playbooks/metron_full_install.yml
index 020cb50..ab72b85 100644
--- a/metron-deployment/ansible/playbooks/metron_full_install.yml
+++ b/metron-deployment/ansible/playbooks/metron_full_install.yml
@@ -15,29 +15,23 @@
# limitations under the License.
#
---
-- hosts: all
- pre_tasks:
- - name: Verify Ansible Version
- fail: msg="Metron Requires Ansible 2.0.0.2, 2.2.2.0, or 2.5.0, current version is {{ ansible_version }}"
- when: "ansible_version.full | version_compare('2.2.2.0', '!=') and ansible_version.full | version_compare('2.0.0.2', '!=') and ansible_version.full | version_compare('2.5.0', '!=')"
-
-- include: metron_build.yml
+- import_playbook: metron_build.yml
tags:
- build
-- include: solr_install.yml
+- import_playbook: solr_install.yml
tags:
- solr
-- include: ambari_install.yml
+- import_playbook: ambari_install.yml
tags:
- ambari
-- include: metron_install.yml
+- import_playbook: metron_install.yml
tags:
- metron
-- include: sensor_install.yml
+- import_playbook: sensor_install.yml
#
# deployment report
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml
index a0ad679..23c4aca 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml
@@ -17,7 +17,8 @@
---
- name: Setup Ambari repo on Ubuntu
- shell: echo "deb {{ ubuntu_ambari_repo }} Ambari main" | sudo tee /etc/apt/sources.list.d/ambari.list
+ shell: echo "deb {{ ubuntu_ambari_repo }} Ambari main" | tee /etc/apt/sources.list.d/ambari.list
+ become: true
- name: Update package cache on Ubuntu
apt: update_cache=yes
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/main.yml b/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
index f3fd905..f92a57e 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
@@ -16,23 +16,20 @@
#
---
-- include: iptables-centos.yml
- static: no
+- include_tasks: iptables-centos.yml
when: ansible_distribution == "CentOS"
-- include: iptables-ubuntu.yml
- static: no
+- include_tasks: iptables-ubuntu.yml
when: ansible_distribution == "Ubuntu"
- include: hostname.yml
-- include: ambari-repo-centos.yml
- static: no
+- include_tasks: ambari-repo-centos.yml
when: ansible_distribution == "CentOS"
-- include: ambari-repo-ubuntu.yml
- static: no
+- include_tasks: ambari-repo-ubuntu.yml
when: ansible_distribution == "Ubuntu"
- include: nodejs.yml
+
- include: logrotate.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml
index 0dbf9f7..ea2067b 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml
@@ -19,16 +19,18 @@
# for CentOS...
#
- name: Install Nodejs repo on CentOS
- shell: curl --silent --location https://rpm.nodesource.com/setup_6.x | bash -
+ shell: curl -sL https://rpm.nodesource.com/setup_6.x | bash -
args:
warn: false
when: ansible_distribution == "CentOS"
+ become: true
#
# for Ubuntu...
#
- name: Install Nodejs repo on Ubuntu
- shell: curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash -
+ shell: curl -sL https://deb.nodesource.com/setup_6.x | bash -
args:
warn: false
when: ansible_distribution == "Ubuntu"
+ become: true
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/ansible/roles/enable-swap/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/enable-swap/tasks/main.yml b/metron-deployment/ansible/roles/enable-swap/tasks/main.yml
index f7d108c..4b159b4 100644
--- a/metron-deployment/ansible/roles/enable-swap/tasks/main.yml
+++ b/metron-deployment/ansible/roles/enable-swap/tasks/main.yml
@@ -26,6 +26,5 @@
- debug: msg="Swap space is NOT yet enabled"
when: swapcheck.rc == 1
-- include: enable-swap.yml
- static: no
+- include_tasks: enable-swap.yml
when: swapcheck.rc != 0
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/ansible/roles/java_jdk/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/java_jdk/tasks/main.yml b/metron-deployment/ansible/roles/java_jdk/tasks/main.yml
index 946da25..8cfea23 100644
--- a/metron-deployment/ansible/roles/java_jdk/tasks/main.yml
+++ b/metron-deployment/ansible/roles/java_jdk/tasks/main.yml
@@ -15,10 +15,8 @@
# limitations under the License.
#
---
-- include: install_jdk_centos.yml
- static: no
+- include_tasks: install_jdk_centos.yml
when: ansible_distribution == "CentOS"
-- include: install_jdk_ubuntu.yml
- static: no
+- include_tasks: install_jdk_ubuntu.yml
when: ansible_distribution == "Ubuntu"
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/ansible/roles/metron-builder/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-builder/tasks/main.yml b/metron-deployment/ansible/roles/metron-builder/tasks/main.yml
index 9636363..56caebc 100644
--- a/metron-deployment/ansible/roles/metron-builder/tasks/main.yml
+++ b/metron-deployment/ansible/roles/metron-builder/tasks/main.yml
@@ -26,10 +26,8 @@
- include: clean.yml
- include: build.yml
-- include: build-rpms.yml
- static: no
+- include_tasks: build-rpms.yml
when: ansible_distribution == "CentOS"
-- include: build-debs.yml
- static: no
+- include_tasks: build-debs.yml
when: ansible_distribution == "Ubuntu"
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/ansible/roles/metron-packages/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/main.yml b/metron-deployment/ansible/roles/metron-packages/tasks/main.yml
index 193a64c..9363a42 100644
--- a/metron-deployment/ansible/roles/metron-packages/tasks/main.yml
+++ b/metron-deployment/ansible/roles/metron-packages/tasks/main.yml
@@ -32,21 +32,17 @@
# repository or whether it can be left empty. copying the packages can be a
# time consuming operation on a large cluster, so it avoided unless required.
#
-- include: copy_packages_centos.yml
- static: no
+- include_tasks: copy_packages_centos.yml
when: ansible_distribution == "CentOS" and packages_required == true
-- include: copy_packages_ubuntu.yml
- static: no
+- include_tasks: copy_packages_ubuntu.yml
when: ansible_distribution == "Ubuntu" and packages_required == true
#
# create the local repository
#
-- include: create_repo_centos.yml
- static: no
+- include_tasks: create_repo_centos.yml
when: ansible_distribution == "CentOS"
-- include: create_repo_ubuntu.yml
- static: no
+- include_tasks: create_repo_ubuntu.yml
when: ansible_distribution == "Ubuntu"
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/development/centos6/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/README.md b/metron-deployment/development/centos6/README.md
index 0e38d5a..e873452 100644
--- a/metron-deployment/development/centos6/README.md
+++ b/metron-deployment/development/centos6/README.md
@@ -29,7 +29,7 @@ Getting Started
The computer used to deploy Apache Metron will need to have the following components installed.
- - [Ansible](https://github.com/ansible/ansible) (2.0.0.2, 2.2.2.0, or 2.5.0)
+ - [Ansible](https://github.com/ansible/ansible) 2.4.0+
- [Docker](https://www.docker.com/community-edition)
- [Vagrant](https://www.vagrantup.com) 2.0+
- [Vagrant Hostmanager Plugin](https://github.com/devopsgroup-io/vagrant-hostmanager)
@@ -53,10 +53,9 @@ Any platform that supports these tools is suitable, but the following instructio
1. Run the following command in a terminal to install all of the required tools.
```
- brew cask install vagrant virtualbox docker
+ brew cask install vagrant virtualbox docker ansible
brew cask install caskroom/versions/java8
brew install maven@3.3 git
- pip install ansible==2.2.2.0
vagrant plugin install vagrant-hostmanager
open /Applications/Docker.app
```
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/development/centos6/Vagrantfile
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/Vagrantfile b/metron-deployment/development/centos6/Vagrantfile
index d0b7051..05823a2 100644
--- a/metron-deployment/development/centos6/Vagrantfile
+++ b/metron-deployment/development/centos6/Vagrantfile
@@ -99,6 +99,6 @@ Vagrant.configure(2) do |config|
ansible.tags = ansibleTags.split(",") if ansibleTags != ''
ansible.skip_tags = ansibleSkipTags.split(",") if ansibleSkipTags != ''
ansible.inventory_path = "ansible/inventory"
- ansible.compatibility_mode = "1.8"
+ ansible.compatibility_mode = "auto"
end
end
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/development/centos6/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/ansible.cfg b/metron-deployment/development/centos6/ansible.cfg
index ade50ae..66f0afd 100644
--- a/metron-deployment/development/centos6/ansible.cfg
+++ b/metron-deployment/development/centos6/ansible.cfg
@@ -20,7 +20,7 @@ library = ../../ansible/extra_modules
roles_path = ../../ansible/roles
pipelining = True
log_path = ./ansible.log
-
+callback_plugins = ../../ansible/callback_plugins
# fix for "ssh throws 'unix domain socket too long' " problem
[ssh_connection]
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/development/centos6/ansible/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/ansible/playbook.yml b/metron-deployment/development/centos6/ansible/playbook.yml
index 562fb1d..433d3b5 100644
--- a/metron-deployment/development/centos6/ansible/playbook.yml
+++ b/metron-deployment/development/centos6/ansible/playbook.yml
@@ -21,4 +21,4 @@
- { role: enable-swap, tags: enable-swap }
- { role: enable-remote-ssh, tags: enable-remote-ssh }
-- include: ../../../ansible/playbooks/metron_full_install.yml
+- import_playbook: ../../../ansible/playbooks/metron_full_install.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/development/fastcapa/centos-7.1/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/development/fastcapa/centos-7.1/ansible.cfg b/metron-deployment/development/fastcapa/centos-7.1/ansible.cfg
index dd78cfd..6ad9241 100644
--- a/metron-deployment/development/fastcapa/centos-7.1/ansible.cfg
+++ b/metron-deployment/development/fastcapa/centos-7.1/ansible.cfg
@@ -14,9 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
[defaults]
host_key_checking = false
library = ../../../ansible/extra_modules
roles_path = ../../../ansible/roles
pipelining = True
+callback_plugins = ../../../ansible/callback_plugins
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/development/fastcapa/centos-7.4/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/development/fastcapa/centos-7.4/ansible.cfg b/metron-deployment/development/fastcapa/centos-7.4/ansible.cfg
index dd78cfd..6ad9241 100644
--- a/metron-deployment/development/fastcapa/centos-7.4/ansible.cfg
+++ b/metron-deployment/development/fastcapa/centos-7.4/ansible.cfg
@@ -14,9 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
[defaults]
host_key_checking = false
library = ../../../ansible/extra_modules
roles_path = ../../../ansible/roles
pipelining = True
+callback_plugins = ../../../ansible/callback_plugins
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/development/ubuntu14/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/README.md b/metron-deployment/development/ubuntu14/README.md
index a3cb89b..494937e 100644
--- a/metron-deployment/development/ubuntu14/README.md
+++ b/metron-deployment/development/ubuntu14/README.md
@@ -29,7 +29,7 @@ Getting Started
The computer used to deploy Apache Metron will need to have the following components installed.
- - [Ansible](https://github.com/ansible/ansible) (2.0.0.2, 2.2.2.0, or 2.5.0)
+ - [Ansible](https://github.com/ansible/ansible) 2.4.0+
- [Docker](https://www.docker.com/community-edition)
- [Vagrant](https://www.vagrantup.com) 2.0+
- [Vagrant Hostmanager Plugin](https://github.com/devopsgroup-io/vagrant-hostmanager)
@@ -53,10 +53,9 @@ Any platform that supports these tools is suitable, but the following instructio
1. Run the following command in a terminal to install all of the required tools.
```
- brew cask install vagrant virtualbox docker
+ brew cask install vagrant virtualbox docker ansible
brew cask install caskroom/versions/java8
brew install maven@3.3 git
- pip install ansible==2.2.2.0
vagrant plugin install vagrant-hostmanager
open /Applications/Docker.app
```
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/development/ubuntu14/Vagrantfile
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/Vagrantfile b/metron-deployment/development/ubuntu14/Vagrantfile
index cfa3cdf..facc3cc 100644
--- a/metron-deployment/development/ubuntu14/Vagrantfile
+++ b/metron-deployment/development/ubuntu14/Vagrantfile
@@ -94,6 +94,6 @@ Vagrant.configure(2) do |config|
ansible.tags = ansibleTags.split(",") if ansibleTags != ''
ansible.skip_tags = ansibleSkipTags.split(",") if ansibleSkipTags != ''
ansible.inventory_path = "ansible/inventory"
- ansible.compatibility_mode = "1.8"
+ ansible.compatibility_mode = "auto"
end
end
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/development/ubuntu14/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/ansible.cfg b/metron-deployment/development/ubuntu14/ansible.cfg
index ade50ae..66f0afd 100644
--- a/metron-deployment/development/ubuntu14/ansible.cfg
+++ b/metron-deployment/development/ubuntu14/ansible.cfg
@@ -20,7 +20,7 @@ library = ../../ansible/extra_modules
roles_path = ../../ansible/roles
pipelining = True
log_path = ./ansible.log
-
+callback_plugins = ../../ansible/callback_plugins
# fix for "ssh throws 'unix domain socket too long' " problem
[ssh_connection]
http://git-wip-us.apache.org/repos/asf/metron/blob/1723a0ea/metron-deployment/development/ubuntu14/ansible/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/ansible/playbook.yml b/metron-deployment/development/ubuntu14/ansible/playbook.yml
index fcbe99c..e4833cc 100644
--- a/metron-deployment/development/ubuntu14/ansible/playbook.yml
+++ b/metron-deployment/development/ubuntu14/ansible/playbook.yml
@@ -22,4 +22,4 @@
roles:
- role: enable-swap
-- include: ../../../ansible/playbooks/metron_full_install.yml
+- import_playbook: ../../../ansible/playbooks/metron_full_install.yml
[24/51] [abbrv] metron git commit: METRON-1771 Update REST endpoints
to support eventually consistent UI updates (merrimanr) closes
apache/metron#1190
Posted by ot...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
index 2f83921..54b5b64 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
@@ -56,7 +56,7 @@ public class SolrUpdateDao implements UpdateDao {
}
@Override
- public void update(Document update, Optional<String> rawIndex) throws IOException {
+ public Document update(Document update, Optional<String> rawIndex) throws IOException {
Document newVersion = update;
// Handle any case where we're given comments in Map form, instead of raw String
Object commentsObj = update.getDocument().get(COMMENTS_FIELD);
@@ -79,10 +79,11 @@ public class SolrUpdateDao implements UpdateDao {
} catch (SolrServerException e) {
throw new IOException(e);
}
+ return newVersion;
}
@Override
- public void batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
+ public Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
// updates with a collection specified
Map<String, Collection<SolrInputDocument>> solrCollectionUpdates = new HashMap<>();
Set<String> collectionsUpdated = new HashSet<>();
@@ -117,18 +118,20 @@ public class SolrUpdateDao implements UpdateDao {
} catch (SolrServerException e) {
throw new IOException(e);
}
+ return updates;
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ public Document addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
- addCommentToAlert(request, latest);
+ return addCommentToAlert(request, latest);
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
- if (latest == null) {
- return;
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ if (latest == null || latest.getDocument() == null) {
+ throw new IOException(String.format("Unable to add comment. Document with guid %s cannot be found.",
+ request.getGuid()));
}
@SuppressWarnings("unchecked")
@@ -149,21 +152,22 @@ public class SolrUpdateDao implements UpdateDao {
Document newVersion = new Document(latest);
newVersion.getDocument().put(COMMENTS_FIELD, commentStrs);
- update(newVersion, Optional.empty());
+ return update(newVersion, Optional.empty());
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request)
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request)
throws IOException {
Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
- removeCommentFromAlert(request, latest);
+ return removeCommentFromAlert(request, latest);
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
throws IOException {
- if (latest == null) {
- return;
+ if (latest == null || latest.getDocument() == null) {
+ throw new IOException(String.format("Unable to remove comment. Document with guid %s cannot be found.",
+ request.getGuid()));
}
@SuppressWarnings("unchecked")
@@ -171,8 +175,8 @@ public class SolrUpdateDao implements UpdateDao {
.get(COMMENTS_FIELD);
// Can't remove anything if there's nothing there
if (commentMap == null) {
- LOG.debug("Provided alert had no comments to be able to remove from");
- return;
+ throw new IOException(String.format("Unable to remove comment. Document with guid %s has no comments.",
+ request.getGuid()));
}
List<Map<String, Object>> originalComments = new ArrayList<>(commentMap);
List<AlertComment> comments = new ArrayList<>();
@@ -186,7 +190,7 @@ public class SolrUpdateDao implements UpdateDao {
.collect(Collectors.toList());
Document newVersion = new Document(latest);
newVersion.getDocument().put(COMMENTS_FIELD, commentsAsJson);
- update(newVersion, Optional.empty());
+ return update(newVersion, Optional.empty());
}
public void convertCommentsToRaw(Map<String,Object> source) {
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
index 43bf1b1..8920d5a 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
@@ -86,31 +86,38 @@ public class SolrMetaAlertDaoTest {
}
@Override
- public void update(Document update, Optional<String> index) {
+ public Document update(Document update, Optional<String> index) {
+ return null;
}
@Override
- public void batchUpdate(Map<Document, Optional<String>> updates) {
+ public Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) {
+ return null;
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) {
+ public Document addCommentToAlert(CommentAddRemoveRequest request) {
+ return null;
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request) {
+ return null;
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ return null;
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ return null;
}
@Override
- public void patch(RetrieveLatestDao dao, PatchRequest request, Optional<Long> timestamp) {
+ public Document patch(RetrieveLatestDao dao, PatchRequest request, Optional<Long> timestamp) {
+ return null;
}
@Override
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
index bed43ae..21fc79b 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
@@ -39,10 +39,12 @@ import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.IndexingConfigurations;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.UpdateDaoTest;
import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
+import org.apache.metron.indexing.dao.update.UpdateDao;
import org.apache.metron.indexing.util.IndexingCacheUtil;
import org.apache.metron.solr.matcher.SolrInputDocumentListMatcher;
import org.apache.metron.solr.matcher.SolrInputDocumentMatcher;
@@ -51,19 +53,19 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.common.SolrInputDocument;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
+/**
+ * This class contains tests specific to the SolrUpdateDao implementation. It also returns the SolrUpdateDao
+ * implementation to be used in UpdateDaoTest. UpdateDaoTest contains a common set of tests that all Dao
+ * implementations must pass.
+ */
@RunWith(PowerMockRunner.class)
@PrepareForTest({CollectionAdminRequest.class})
-public class SolrUpdateDaoTest {
-
- @Rule
- public final ExpectedException exception = ExpectedException.none();
+public class SolrUpdateDaoTest extends UpdateDaoTest {
private SolrClient client;
private SolrRetrieveLatestDao solrRetrieveLatestDao;
@@ -238,4 +240,9 @@ public class SolrUpdateDaoTest {
latest.getDocument().put("project", "metron");
assertEquals(actual, latest);
}
+
+ @Override
+ public UpdateDao getUpdateDao() {
+ return solrUpdateDao;
+ }
}
[48/51] [abbrv] metron git commit: METRON-1833: Management UI
incorrectly displaying sensor topology latency units as seconds instead of
millis (mmiklavc via mmiklavc) closes apache/metron#1241
Posted by ot...@apache.org.
METRON-1833: Management UI incorrectly displaying sensor topology latency units as seconds instead of millis (mmiklavc via mmiklavc) closes apache/metron#1241
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0c0602c7
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0c0602c7
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0c0602c7
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 0c0602c75d9660717920a547bcb9e76d58b6571e
Parents: d44a392
Author: mmiklavc <mi...@gmail.com>
Authored: Mon Oct 22 12:54:26 2018 -0600
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Mon Oct 22 12:54:26 2018 -0600
----------------------------------------------------------------------
.../sensor-parser-config-readonly.component.ts | 2 +-
.../sensor-parser-list/sensor-parser-list.component.spec.ts | 2 +-
.../app/sensors/sensor-parser-list/sensor-parser-list.component.ts | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/0c0602c7/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts b/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts
index 5db6d45..7d41003 100644
--- a/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts
+++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts
@@ -156,7 +156,7 @@ export class SensorParserConfigReadonlyComponent implements OnInit {
getTopologyStatus(key: string): string {
if (key === 'latency') {
- return this.topologyStatus.latency >= 0 ? (this.topologyStatus.latency + 's') : '-';
+ return this.topologyStatus.latency >= 0 ? (this.topologyStatus.latency + 'ms') : '-';
} else if (key === 'throughput') {
return this.topologyStatus.throughput >= 0 ? ((Math.round(this.topologyStatus.throughput * 100) / 100) + 'kb/s') : '-';
} else if (key === 'emitted') {
http://git-wip-us.apache.org/repos/asf/metron/blob/0c0602c7/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts
index 205d885..fb2a175 100644
--- a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts
+++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts
@@ -720,7 +720,7 @@ describe('Component: SensorParserList', () => {
component.updateSensorStatus();
expect(component.sensors[0]['status']).toEqual('Running');
- expect(component.sensors[0]['latency']).toEqual('10s');
+ expect(component.sensors[0]['latency']).toEqual('10ms');
expect(component.sensors[0]['throughput']).toEqual('23kb/s');
component.sensorsStatus[0].status = 'KILLED';
http://git-wip-us.apache.org/repos/asf/metron/blob/0c0602c7/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts
index 1129914..2694ab4 100644
--- a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts
+++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts
@@ -163,7 +163,7 @@ export class SensorParserListComponent implements OnInit {
sensor['status'] = 'Stopped';
}
- sensor['latency'] = status && status.status === 'ACTIVE' ? (status.latency + 's') : '-';
+ sensor['latency'] = status && status.status === 'ACTIVE' ? (status.latency + 'ms') : '-';
sensor['throughput'] = status && status.status === 'ACTIVE' ? (Math.round(status.throughput * 100) / 100) + 'kb/s' : '-';
}
}
[46/51] [abbrv] metron git commit: METRON-1831 Project Version
Substitution Not Working (nickwallen) closes apache/metron#1243
Posted by ot...@apache.org.
METRON-1831 Project Version Substitution Not Working (nickwallen) closes apache/metron#1243
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2531c3e1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2531c3e1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2531c3e1
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 2531c3e19e6c2887a2242da830d7178e6973b8ab
Parents: bf32fef
Author: nickwallen <ni...@nickallen.org>
Authored: Sat Oct 20 11:48:38 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Sat Oct 20 11:48:38 2018 -0400
----------------------------------------------------------------------
metron-platform/metron-common/src/main/scripts/stellar | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/2531c3e1/metron-platform/metron-common/src/main/scripts/stellar
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/scripts/stellar b/metron-platform/metron-common/src/main/scripts/stellar
index 34097f2..690a9f0 100644
--- a/metron-platform/metron-common/src/main/scripts/stellar
+++ b/metron-platform/metron-common/src/main/scripts/stellar
@@ -33,8 +33,9 @@ if [ -f "$METRON_SYSCONFIG" ]; then
source $METRON_SYSCONFIG
fi
-export METRON_VERSION="${METRON_VERSION:-${project.version}}"
-export METRON_HOME="${METRON_HOME:-/usr/metron/$METRON_VERSION}"
+# treat unset vars as an error; METRON_HOME
+set -u
+
export HBASE_CONFIGS=$(hbase classpath)
export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar)
export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar)
[33/51] [abbrv] metron git commit: METRON-1811 Alert Search Fails
When Sorting by Alert Status (merrimanr) closes apache/metron#1231
Posted by ot...@apache.org.
METRON-1811 Alert Search Fails When Sorting by Alert Status (merrimanr) closes apache/metron#1231
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/747220f0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/747220f0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/747220f0
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 747220f000188557bfa860b48f6eb4bc49758d4a
Parents: 3467ffd
Author: merrimanr <me...@gmail.com>
Authored: Tue Oct 9 15:33:39 2018 -0500
Committer: rmerriman <me...@gmail.com>
Committed: Tue Oct 9 15:33:39 2018 -0500
----------------------------------------------------------------------
.../CURRENT/package/files/bro_index.template | 3 +
.../CURRENT/package/files/snort_index.template | 3 +
.../CURRENT/package/files/yaf_index.template | 3 +
metron-platform/metron-elasticsearch/README.md | 20 +-
.../ElasticsearchSearchIntegrationTest.java | 211 ++++++-------------
.../indexing/dao/SearchIntegrationTest.java | 34 +--
.../src/main/config/schema/bro/schema.xml | 1 +
.../src/main/config/schema/snort/schema.xml | 1 +
.../src/main/config/schema/yaf/schema.xml | 1 +
.../integration/SolrSearchIntegrationTest.java | 8 +-
10 files changed, 112 insertions(+), 173 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
index 17ad4d2..48e76c7 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
@@ -133,6 +133,9 @@
"source:type": {
"type": "keyword"
},
+ "alert_status": {
+ "type": "keyword"
+ },
/*
* Widely-used Bro fields (potentially renamed during Metron ingest)
*/
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
index f7c6e59..139dbc9 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
@@ -123,6 +123,9 @@
"source:type": {
"type": "keyword"
},
+ "alert_status": {
+ "type": "keyword"
+ },
"ip_dst_addr": {
"type": "ip"
},
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
index f4093ba..4a3e5f9 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
@@ -123,6 +123,9 @@
"source:type": {
"type": "keyword"
},
+ "alert_status": {
+ "type": "keyword"
+ },
"ip_dst_addr": {
"type": "ip"
},
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-elasticsearch/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/README.md b/metron-platform/metron-elasticsearch/README.md
index d889e27..177412e 100644
--- a/metron-platform/metron-elasticsearch/README.md
+++ b/metron-platform/metron-elasticsearch/README.md
@@ -269,9 +269,27 @@ Notes on other settings for types in ES
* [https://www.elastic.co/guide/en/elasticsearch/reference/5.6/breaking_50_mapping_changes.html](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/breaking_50_mapping_changes.html)
* [https://www.elastic.co/blog/strings-are-dead-long-live-strings](https://www.elastic.co/blog/strings-are-dead-long-live-strings)
+### Metron Properties
+
+Metron depends on some internal fields being defined in sensor templates. A field is defined in Elasticsearch by adding an entry to the `properties` section of the template:
+```
+"properties": {
+ "metron_field": {
+ "type": "keyword"
+ }
+}
+```
+
+The following is a list of properties that need to be defined along with their type:
+* source:type - keyword
+* alert_status - keyword
+* metron_alert - nested
+
## Using Metron with Elasticsearch 5.6.2
-There is a requirement that all sensors templates have a nested `metron_alert` field defined. This field is a dummy field. See [Ignoring Unmapped Fields](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html#_ignoring_unmapped_fields) for more information
+Although infrequent sometimes an internal field is added in Metron and existing templates must be updated. The following steps outlines how to do this, using `metron_alert` as an example.
+
+With the addition of the meta alert feature, there is a requirement that all sensors templates have a nested `metron_alert` field defined. This field is a dummy field. See [Ignoring Unmapped Fields](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html#_ignoring_unmapped_fields) for more information
Without this field, an error will be thrown during ALL searches (including from UIs, resulting in no alerts being found for any sensor). This error will be found in the REST service's logs.
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 8071e68..1d2d48e 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -19,6 +19,7 @@ package org.apache.metron.elasticsearch.integration;
import java.io.File;
+import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -55,136 +56,12 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
private static String indexDir = "target/elasticsearch_search";
private static String dateFormat = "yyyy.MM.dd.HH";
+ private static String broTemplatePath = "../../metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template";
+ private static String snortTemplatePath = "../../metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template";
private static final int MAX_RETRIES = 10;
private static final int SLEEP_MS = 500;
protected static IndexDao dao;
- /**
- * {
- * "bro_doc": {
- * "properties": {
- * "source:type": {
- * "type": "text",
- * "fielddata" : "true"
- * },
- * "guid" : {
- * "type" : "keyword"
- * },
- * "ip_src_addr": {
- * "type": "ip"
- * },
- * "ip_src_port": {
- * "type": "integer"
- * },
- * "long_field": {
- * "type": "long"
- * },
- * "timestamp": {
- * "type": "date",
- * "format": "epoch_millis"
- * },
- * "latitude" : {
- * "type": "float"
- * },
- * "score": {
- * "type": "double"
- * },
- * "is_alert": {
- * "type": "boolean"
- * },
- * "location_point": {
- * "type": "geo_point"
- * },
- * "bro_field": {
- * "type": "text",
- * "fielddata" : "true"
- * },
- * "ttl": {
- * "type": "text",
- * "fielddata" : "true"
- * },
- * "alert": {
- * "type": "nested"
- * }
- * }
- * }
- * }
- */
- @Multiline
- private static String broTypeMappings;
-
- /**
- * {
- * "snort_doc": {
- * "properties": {
- * "source:type": {
- * "type": "text",
- * "fielddata" : "true"
- * },
- * "guid" : {
- * "type" : "keyword"
- * },
- * "ip_src_addr": {
- * "type": "ip"
- * },
- * "ip_src_port": {
- * "type": "integer"
- * },
- * "long_field": {
- * "type": "long"
- * },
- * "timestamp": {
- * "type": "date",
- * "format": "epoch_millis"
- * },
- * "latitude" : {
- * "type": "float"
- * },
- * "score": {
- * "type": "double"
- * },
- * "is_alert": {
- * "type": "boolean"
- * },
- * "location_point": {
- * "type": "geo_point"
- * },
- * "snort_field": {
- * "type": "integer"
- * },
- * "ttl": {
- * "type": "integer"
- * },
- * "alert": {
- * "type": "nested"
- * },
- * "threat:triage:score": {
- * "type": "float"
- * }
- * }
- * }
- * }
- */
- @Multiline
- private static String snortTypeMappings;
-
- /**
- * {
- * "bro_doc_default": {
- * "dynamic_templates": [{
- * "strings": {
- * "match_mapping_type": "string",
- * "mapping": {
- * "type": "text"
- * }
- * }
- * }]
- * }
- * }
- */
- @Multiline
- private static String broDefaultStringMappings;
-
@BeforeClass
public static void setup() throws Exception {
indexComponent = startIndex();
@@ -220,13 +97,17 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
return es;
}
- protected static void loadTestData() throws ParseException {
+ protected static void loadTestData() throws ParseException, IOException {
ElasticSearchComponent es = (ElasticSearchComponent) indexComponent;
+
+ JSONObject broTemplate = JSONUtils.INSTANCE.load(new File(broTemplatePath), JSONObject.class);
+ addTestFieldMappings(broTemplate, "bro_doc");
es.getClient().admin().indices().prepareCreate("bro_index_2017.01.01.01")
- .addMapping("bro_doc", broTypeMappings)
- .addMapping("bro_doc_default", broDefaultStringMappings).get();
+ .addMapping("bro_doc", JSONUtils.INSTANCE.toJSON(broTemplate.get("mappings"), false)).get();
+ JSONObject snortTemplate = JSONUtils.INSTANCE.load(new File(snortTemplatePath), JSONObject.class);
+ addTestFieldMappings(snortTemplate, "snort_doc");
es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02")
- .addMapping("snort_doc", snortTypeMappings).get();
+ .addMapping("snort_doc", JSONUtils.INSTANCE.toJSON(snortTemplate.get("mappings"), false)).get();
BulkRequestBuilder bulkRequest = es.getClient().prepareBulk()
.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
@@ -258,6 +139,27 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
}
}
+ /**
+ * Add test fields to a template with defined types in case they are not defined in the sensor template shipped with Metron.
+ * This is useful for testing certain cases, for example faceting on fields of various types.
+ * @param template
+ * @param docType
+ */
+ private static void addTestFieldMappings(JSONObject template, String docType) {
+ Map mappings = (Map) template.get("mappings");
+ Map docTypeJSON = (Map) mappings.get(docType);
+ Map properties = (Map) docTypeJSON.get("properties");
+ Map<String, String> longType = new HashMap<>();
+ longType.put("type", "long");
+ properties.put("long_field", longType);
+ Map<String, String> floatType = new HashMap<>();
+ floatType.put("type", "float");
+ properties.put("latitude", floatType);
+ Map<String, String> doubleType = new HashMap<>();
+ doubleType.put("type", "double");
+ properties.put("score", doubleType);
+ }
+
@Test
public void bad_facet_query_throws_exception() throws Exception {
thrown.expect(InvalidSearchException.class);
@@ -273,11 +175,11 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
// getColumnMetadata with only bro
{
Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro"));
- Assert.assertEquals(13, fieldTypes.size());
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl"));
+ Assert.assertEquals(262, fieldTypes.size());
+ Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("method"));
+ Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("ttl"));
Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
+ Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("source:type"));
Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
@@ -285,19 +187,18 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
- Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl"));
- Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("location_point"));
+ Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("ttl"));
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("metron_alert"));
}
// getColumnMetadata with only snort
{
Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort"));
- Assert.assertEquals(14, fieldTypes.size());
- Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
+ Assert.assertEquals(32, fieldTypes.size());
+ Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("sig_generator"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl"));
Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
+ Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("source:type"));
Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
@@ -305,32 +206,40 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
- Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("location_point"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl"));
- Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert"));
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("metron_alert"));
}
}
@Override
public void returns_column_data_for_multiple_indices() throws Exception {
Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort"));
- Assert.assertEquals(15, fieldTypes.size());
+ Assert.assertEquals(277, fieldTypes.size());
+
+ // Ensure internal Metron fields are properly defined
Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
+ Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("source:type"));
+ Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score"));
+ Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("alert_status"));
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("metron_alert"));
+
Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
Assert.assertEquals(FieldType.DATE, fieldTypes.get("timestamp"));
Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
+ Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("suppress_for"));
Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
- Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
- Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
- //NOTE: This is because the field is in both bro and snort and they have different types.
+
+ // Ensure a field defined only in bro is included
+ Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("method"));
+ // Ensure a field defined only in snort is included
+ Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("sig_generator"));
+ // Ensure fields in both bro and snort have type OTHER because they have different types
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("ttl"));
- Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score"));
- Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert"));
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("msg"));
}
@Test
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index 2e1968a..cfe5752 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -46,11 +46,11 @@ import org.junit.rules.ExpectedException;
public abstract class SearchIntegrationTest {
/**
* [
- * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "ttl": "data 1", "guid":"bro_1"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "ttl": "data 2", "guid":"bro_2"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "ttl": "data 3", "guid":"bro_3"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "ttl": "data 4", "guid":"bro_4"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "ttl": "data 5", "guid":"bro_5"}
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "method": "bro data 1", "ttl": "data 1", "guid":"bro_1"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "method": "bro data 2", "ttl": "data 2", "guid":"bro_2"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "method": "bro data 3", "ttl": "data 3", "guid":"bro_3"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "method": "bro data 4", "ttl": "data 4", "guid":"bro_4"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "method": "bro data 5", "ttl": "data 5", "guid":"bro_5"}
* ]
*/
@Multiline
@@ -58,11 +58,11 @@ public abstract class SearchIntegrationTest {
/**
* [
- * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "ttl": 1, "guid":"snort_1", "threat:triage:score":10.0},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "ttl": 2, "guid":"snort_2", "threat:triage:score":20.0},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "ttl": 3, "guid":"snort_3"},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "ttl": 4, "guid":"snort_4"},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "ttl": 5, "guid":"snort_5"}
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "sig_generator": "sig_generator 1", "ttl": 1, "guid":"snort_1", "threat:triage:score":10.0},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "sig_generator": "sig_generator 2", "ttl": 2, "guid":"snort_2", "threat:triage:score":20.0},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "sig_generator": "sig_generator 3", "ttl": 3, "guid":"snort_3"},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "sig_generator": "sig_generator 4", "ttl": 4, "guid":"snort_4"},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "sig_generator": "sig_generator 5", "ttl": 5, "guid":"snort_5"}
* ]
*/
@Multiline
@@ -272,7 +272,7 @@ public abstract class SearchIntegrationTest {
/**
* {
- * "facetFields": ["snort_field"],
+ * "facetFields": ["sig_generator"],
* "indices": ["bro", "snort"],
* "query": "*:*",
* "from": 0,
@@ -698,14 +698,14 @@ public abstract class SearchIntegrationTest {
Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
Assert.assertEquals(1, facetCounts.size());
- Map<String, Long> snortFieldCounts = facetCounts.get("snort_field");
+ Map<String, Long> snortFieldCounts = facetCounts.get("sig_generator");
Assert.assertEquals(5, snortFieldCounts.size());
- Assert.assertEquals(1L, snortFieldCounts.get("50").longValue());
- Assert.assertEquals(1L, snortFieldCounts.get("40").longValue());
- Assert.assertEquals(1L, snortFieldCounts.get("30").longValue());
- Assert.assertEquals(1L, snortFieldCounts.get("20").longValue());
- Assert.assertEquals(1L, snortFieldCounts.get("10").longValue());
+ Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 5").longValue());
+ Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 4").longValue());
+ Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 3").longValue());
+ Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 2").longValue());
+ Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 1").longValue());
response.getFacetCounts();
}
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
index ea9f6d3..6be76a0 100644
--- a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
@@ -34,6 +34,7 @@
* Metron-specific fields
-->
<field name="source.type" type="string" indexed="true" stored="true" />
+ <field name="alert_status" type="string" indexed="true" stored="true" />
<field name="timestamp" type="timestamp" indexed="true" stored="true" />
<field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" />
<uniqueKey>guid</uniqueKey>
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
index 84855df..3c57574 100644
--- a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
@@ -22,6 +22,7 @@
<!-- Metron specific fields -->
<field name="timestamp" type="timestamp" indexed="true" stored="true" />
<field name="source.type" type="string" indexed="true" stored="true" />
+ <field name="alert_status" type="string" indexed="true" stored="true" />
<field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" />
<uniqueKey>guid</uniqueKey>
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
index 5555a14..37e5f12 100644
--- a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
@@ -22,6 +22,7 @@
<!-- Metron specific fields -->
<field name="timestamp" type="timestamp" indexed="true" stored="true" />
<field name="source.type" type="string" indexed="true" stored="true" />
+ <field name="alert_status" type="string" indexed="true" stored="true" />
<field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" />
<uniqueKey>guid</uniqueKey>
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
index 4390fd1..84f2222 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
@@ -133,7 +133,7 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp"));
// Bro only field in the dynamic catch all
- Assert.assertEquals(FieldType.OTHER, fieldTypes.get("bro_field"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("method"));
// A field is in both bro and snort and they have different types.
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl"));
@@ -169,7 +169,7 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp"));
// Snort only field in the dynamic catch all
- Assert.assertEquals(FieldType.OTHER, fieldTypes.get("snort_field"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("sig_generator"));
// A field is in both bro and snort and they have different types.
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl"));
@@ -211,10 +211,10 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp"));
// Bro only field in the dynamic catch all
- Assert.assertEquals(FieldType.OTHER, fieldTypes.get("bro_field"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("method"));
// Snort only field in the dynamic catch all
- Assert.assertEquals(FieldType.OTHER, fieldTypes.get("snort_field"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("sig_generator"));
// A field is in both bro and snort and they have different types.
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("ttl"));
[44/51] [abbrv] metron git commit: METRON-1681 Decouple the
ParserBolt from the Parse execution logic (merrimanr) closes
apache/metron#1213
Posted by ot...@apache.org.
METRON-1681 Decouple the ParserBolt from the Parse execution logic (merrimanr) closes apache/metron#1213
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/28542ad6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/28542ad6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/28542ad6
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 28542ad64cf63f17b728b4b1c0e995a8973767f7
Parents: 08f3de0
Author: merrimanr <me...@gmail.com>
Authored: Thu Oct 18 13:59:52 2018 -0500
Committer: rmerriman <me...@gmail.com>
Committed: Thu Oct 18 13:59:52 2018 -0500
----------------------------------------------------------------------
.../impl/SensorParserConfigServiceImpl.java | 51 +-
.../parsers/DefaultParserRunnerResults.java | 71 ++
.../org/apache/metron/parsers/GrokParser.java | 3 +-
.../org/apache/metron/parsers/ParserRunner.java | 60 ++
.../apache/metron/parsers/ParserRunnerImpl.java | 322 +++++++
.../metron/parsers/ParserRunnerResults.java | 33 +
.../apache/metron/parsers/bolt/ParserBolt.java | 381 +++-----
.../parsers/filters/BroMessageFilter.java | 2 +-
.../metron/parsers/filters/StellarFilter.java | 2 +-
.../parsers/interfaces/MessageFilter.java | 2 +-
.../parsers/interfaces/MessageParser.java | 27 +-
.../interfaces/MultilineMessageParser.java | 51 --
.../metron/parsers/syslog/Syslog5424Parser.java | 4 +-
.../parsers/topology/ParserComponent.java | 56 ++
.../parsers/topology/ParserComponents.java | 67 --
.../parsers/topology/ParserTopologyBuilder.java | 39 +-
.../org/apache/metron/filters/FiltersTest.java | 4 +-
.../metron/parsers/MessageParserTest.java | 108 ++-
.../metron/parsers/ParserRunnerImplTest.java | 390 +++++++++
.../metron/parsers/bolt/ParserBoltTest.java | 859 ++++++-------------
.../parsers/integration/ParserDriver.java | 60 +-
21 files changed, 1481 insertions(+), 1111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
index 4cd272e..d0e4b3d 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -20,12 +20,10 @@ package org.apache.metron.rest.service.impl;
import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
import com.fasterxml.jackson.databind.ObjectMapper;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.fs.Path;
import org.apache.metron.common.configuration.ConfigurationType;
@@ -35,18 +33,14 @@ import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.parsers.interfaces.MessageParserResult;
-import org.apache.metron.parsers.interfaces.MultilineMessageParser;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.ParseMessageRequest;
import org.apache.metron.rest.service.GrokService;
import org.apache.metron.rest.service.SensorParserConfigService;
import org.apache.metron.rest.util.ParserIndex;
-import org.apache.metron.common.zookeeper.ZKConfigurationsCache;
import org.apache.zookeeper.KeeperException;
import org.json.simple.JSONObject;
-import org.reflections.Reflections;
-import org.reflections.util.ConfigurationBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -141,53 +135,13 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
} else if (sensorParserConfig.getParserClassName() == null) {
throw new RestException("SensorParserConfig must have a parserClassName");
} else {
- MultilineMessageParser<JSONObject> parser;
- Object parserObject;
+ MessageParser<JSONObject> parser;
try {
- parserObject = Class.forName(sensorParserConfig.getParserClassName())
+ parser = (MessageParser<JSONObject>) Class.forName(sensorParserConfig.getParserClassName())
.newInstance();
} catch (Exception e) {
throw new RestException(e.toString(), e.getCause());
}
-
- if (!(parserObject instanceof MultilineMessageParser)) {
- parser = new MultilineMessageParser<JSONObject>() {
-
- @Override
- @SuppressWarnings("unchecked")
- public void configure(Map<String, Object> config) {
- ((MessageParser<JSONObject>)parserObject).configure(config);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void init() {
- ((MessageParser<JSONObject>)parserObject).init();
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public boolean validate(JSONObject message) {
- return ((MessageParser<JSONObject>)parserObject).validate(message);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public List<JSONObject> parse(byte[] message) {
- return ((MessageParser<JSONObject>)parserObject).parse(message);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public Optional<List<JSONObject>> parseOptional(byte[] message) {
- return ((MessageParser<JSONObject>)parserObject).parseOptional(message);
- }
- };
- } else {
- parser = (MultilineMessageParser<JSONObject>)parserObject;
- }
-
-
Path temporaryGrokPath = null;
if (isGrokConfig(sensorParserConfig)) {
String name = parseMessageRequest.getSensorParserConfig().getSensorTopic();
@@ -195,7 +149,6 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
sensorParserConfig.getParserConfig()
.put(MetronRestConstants.GROK_PATH_KEY, new Path(temporaryGrokPath, name).toString());
}
-
parser.configure(sensorParserConfig.getParserConfig());
parser.init();
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/DefaultParserRunnerResults.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/DefaultParserRunnerResults.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/DefaultParserRunnerResults.java
new file mode 100644
index 0000000..79a9b5d
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/DefaultParserRunnerResults.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers;
+
+import org.apache.metron.common.error.MetronError;
+import org.json.simple.JSONObject;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Default implementation of ParserRunnerResults.
+ */
+public class DefaultParserRunnerResults implements ParserRunnerResults<JSONObject> {
+
+ private List<JSONObject> messages = new ArrayList<>();
+ private List<MetronError> errors = new ArrayList<>();
+
+ public List<JSONObject> getMessages() {
+ return messages;
+ }
+
+ public List<MetronError> getErrors() {
+ return errors;
+ }
+
+ public void addMessage(JSONObject message) {
+ this.messages.add(message);
+ }
+
+ public void addError(MetronError error) {
+ this.errors.add(error);
+ }
+
+ public void addErrors(List<MetronError> errors) {
+ this.errors.addAll(errors);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ParserRunnerResults parserResult = (ParserRunnerResults) o;
+ return Objects.equals(messages, parserResult.getMessages()) &&
+ Objects.equals(errors, parserResult.getErrors());
+ }
+
+ @Override
+ public int hashCode() {
+ int result = messages != null ? messages.hashCode() : 0;
+ result = 31 * result + (errors != null ? errors.hashCode() : 0);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
index a81149d..6bdfb81 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.metron.common.Constants;
import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.parsers.interfaces.MessageParserResult;
-import org.apache.metron.parsers.interfaces.MultilineMessageParser;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,7 +53,7 @@ import java.util.Optional;
import java.util.TimeZone;
-public class GrokParser implements MultilineMessageParser<JSONObject>, Serializable {
+public class GrokParser implements MessageParser<JSONObject>, Serializable {
protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunner.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunner.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunner.java
new file mode 100644
index 0000000..f9123b1
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunner.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers;
+
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.message.metadata.RawMessage;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.apache.metron.stellar.dsl.Context;
+
+import java.util.List;
+import java.util.Set;
+import java.util.function.Supplier;
+
+/**
+ * A ParserRunner is responsible for initializing MessageParsers and parsing messages with the appropriate MessageParser.
+ * The information needed to initialize a MessageParser is supplied by the parser config supplier. After the parsers
+ * are initialized, the execute method can then be called for each message and will return a ParserRunnerResults object
+ * that contains a list of parsed messages and/or a list of errors.
+ * @param <T> The type of a successfully parsed message.
+ */
+public interface ParserRunner<T> {
+
+ /**
+ * Return a list of all sensor types that can be parsed with this ParserRunner.
+ * @return Sensor types
+ */
+ Set<String> getSensorTypes();
+
+ /**
+ *
+ * @param parserConfigSupplier Supplies parser configurations
+ * @param stellarContext Stellar context used to apply Stellar functions during field transformations
+ */
+ void init(Supplier<ParserConfigurations> parserConfigSupplier, Context stellarContext);
+
+ /**
+ * Parses a message and either returns the message or an error.
+ * @param sensorType Sensor type of the message
+ * @param rawMessage Raw message including metadata
+ * @param parserConfigurations Parser configurations
+ * @return ParserRunnerResults containing a list of messages and a list of errors
+ */
+ ParserRunnerResults<T> execute(String sensorType, RawMessage rawMessage, ParserConfigurations parserConfigurations);
+
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunnerImpl.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunnerImpl.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunnerImpl.java
new file mode 100644
index 0000000..a986db7
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunnerImpl.java
@@ -0,0 +1,322 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.FieldTransformer;
+import org.apache.metron.common.configuration.FieldValidator;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.error.MetronError;
+import org.apache.metron.common.message.metadata.RawMessage;
+import org.apache.metron.common.utils.ReflectionUtils;
+import org.apache.metron.parsers.filters.Filters;
+import org.apache.metron.parsers.interfaces.MessageFilter;
+import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.apache.metron.parsers.topology.ParserComponent;
+import org.apache.metron.stellar.dsl.Context;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+/**
+ * The default implemention of a ParserRunner.
+ */
+public class ParserRunnerImpl implements ParserRunner<JSONObject>, Serializable {
+
+ class ProcessResult {
+
+ private JSONObject message;
+ private MetronError error;
+
+ public ProcessResult(JSONObject message) {
+ this.message = message;
+ }
+
+ public ProcessResult(MetronError error) {
+ this.error = error;
+ }
+
+ public JSONObject getMessage() {
+ return message;
+ }
+
+ public MetronError getError() {
+ return error;
+ }
+
+ public boolean isError() {
+ return error != null;
+ }
+ }
+
+ protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ protected transient Consumer<ParserRunnerResults> onSuccess;
+ protected transient Consumer<MetronError> onError;
+
+ private HashSet<String> sensorTypes;
+ private Map<String, ParserComponent> sensorToParserComponentMap;
+
+ // Stellar variables
+ private transient Context stellarContext;
+
+ public ParserRunnerImpl(HashSet<String> sensorTypes) {
+ this.sensorTypes = sensorTypes;
+ }
+
+ public Map<String, ParserComponent> getSensorToParserComponentMap() {
+ return sensorToParserComponentMap;
+ }
+
+ public void setSensorToParserComponentMap(Map<String, ParserComponent> sensorToParserComponentMap) {
+ this.sensorToParserComponentMap = sensorToParserComponentMap;
+ }
+
+ public Context getStellarContext() {
+ return stellarContext;
+ }
+
+ @Override
+ public Set<String> getSensorTypes() {
+ return sensorTypes;
+ }
+
+ @Override
+ public void init(Supplier<ParserConfigurations> parserConfigSupplier, Context stellarContext) {
+ if (parserConfigSupplier == null) {
+ throw new IllegalStateException("A parser config supplier must be set before initializing the ParserRunner.");
+ }
+ if (stellarContext == null) {
+ throw new IllegalStateException("A stellar context must be set before initializing the ParserRunner.");
+ }
+ this.stellarContext = stellarContext;
+ initializeParsers(parserConfigSupplier);
+ }
+
+ /**
+ * Parses messages with the appropriate MessageParser based on sensor type. The resulting list of messages are then
+ * post-processed and added to the ParserRunnerResults message list. Any errors that happen during post-processing are
+ * added to the ParserRunnerResults error list. Any exceptions (including a master exception) thrown by the MessageParser
+ * are also added to the ParserRunnerResults error list.
+ *
+ * @param sensorType Sensor type of the message
+ * @param rawMessage Raw message including metadata
+ * @param parserConfigurations Parser configurations
+ * @return ParserRunnerResults containing a list of messages and a list of errors
+ */
+ @Override
+ public ParserRunnerResults<JSONObject> execute(String sensorType, RawMessage rawMessage, ParserConfigurations parserConfigurations) {
+ DefaultParserRunnerResults parserRunnerResults = new DefaultParserRunnerResults();
+ SensorParserConfig sensorParserConfig = parserConfigurations.getSensorParserConfig(sensorType);
+ if (sensorParserConfig != null) {
+ MessageParser<JSONObject> parser = sensorToParserComponentMap.get(sensorType).getMessageParser();
+ Optional<MessageParserResult<JSONObject>> optionalMessageParserResult = parser.parseOptionalResult(rawMessage.getMessage());
+ if (optionalMessageParserResult.isPresent()) {
+ MessageParserResult<JSONObject> messageParserResult = optionalMessageParserResult.get();
+
+ // Process each message returned from the MessageParser
+ messageParserResult.getMessages().forEach(message -> {
+ Optional<ProcessResult> processResult = processMessage(sensorType, message, rawMessage, parser, parserConfigurations);
+ if (processResult.isPresent()) {
+ if (processResult.get().isError()) {
+ parserRunnerResults.addError(processResult.get().getError());
+ } else {
+ parserRunnerResults.addMessage(processResult.get().getMessage());
+ }
+ }
+ });
+
+ // If a master exception is thrown by the MessageParser, wrap it with a MetronError and add it to the list of errors
+ messageParserResult.getMasterThrowable().ifPresent(throwable -> parserRunnerResults.addError(new MetronError()
+ .withErrorType(Constants.ErrorType.PARSER_ERROR)
+ .withThrowable(throwable)
+ .withSensorType(Collections.singleton(sensorType))
+ .addRawMessage(rawMessage.getMessage())));
+
+ // If exceptions are thrown by the MessageParser, wrap them with MetronErrors and add them to the list of errors
+ parserRunnerResults.addErrors(messageParserResult.getMessageThrowables().entrySet().stream().map(entry -> new MetronError()
+ .withErrorType(Constants.ErrorType.PARSER_ERROR)
+ .withThrowable(entry.getValue())
+ .withSensorType(Collections.singleton(sensorType))
+ .addRawMessage(entry.getKey())).collect(Collectors.toList()));
+ }
+ } else {
+ throw new IllegalStateException(String.format("Could not execute parser. Cannot find configuration for sensor %s.",
+ sensorType));
+ }
+ return parserRunnerResults;
+ }
+
+ /**
+ * Initializes MessageParsers and MessageFilters for sensor types configured in this ParserRunner. Objects are created
+ * using reflection and the MessageParser configure and init methods are called.
+ * @param parserConfigSupplier Parser configurations
+ */
+ private void initializeParsers(Supplier<ParserConfigurations> parserConfigSupplier) {
+ LOG.info("Initializing parsers...");
+ sensorToParserComponentMap = new HashMap<>();
+ for(String sensorType: sensorTypes) {
+ if (parserConfigSupplier.get().getSensorParserConfig(sensorType) == null) {
+ throw new IllegalStateException(String.format("Could not initialize parsers. Cannot find configuration for sensor %s.",
+ sensorType));
+ }
+
+ SensorParserConfig parserConfig = parserConfigSupplier.get().getSensorParserConfig(sensorType);
+
+ LOG.info("Creating parser for sensor {} with parser class = {} and filter class = {} ",
+ sensorType, parserConfig.getParserClassName(), parserConfig.getFilterClassName());
+
+ // create message parser
+ MessageParser<JSONObject> parser = ReflectionUtils
+ .createInstance(parserConfig.getParserClassName());
+
+ // create message filter
+ MessageFilter<JSONObject> filter = null;
+ parserConfig.getParserConfig().putIfAbsent("stellarContext", stellarContext);
+ if (!StringUtils.isEmpty(parserConfig.getFilterClassName())) {
+ filter = Filters.get(
+ parserConfig.getFilterClassName(),
+ parserConfig.getParserConfig()
+ );
+ }
+
+ parser.configure(parserConfig.getParserConfig());
+ parser.init();
+ sensorToParserComponentMap.put(sensorType, new ParserComponent(parser, filter));
+ }
+ }
+
+ /**
+ * Post-processes parsed messages by:
+ * <ul>
+ * <li>Applying field transformations defined in the sensor parser config</li>
+ * <li>Filtering messages using the configured MessageFilter class</li>
+ * <li>Validating messages using the MessageParser validate method</li>
+ * </ul>
+ * If a message is successfully processed a message is returned in a ProcessResult. If a message fails
+ * validation, a MetronError object is created and returned in a ProcessResult. If a message is
+ * filtered out an empty Optional is returned.
+ *
+ * @param sensorType Sensor type of the message
+ * @param message Message parsed by the MessageParser
+ * @param rawMessage Raw message including metadata
+ * @param parser MessageParser for the sensor type
+ * @param parserConfigurations Parser configurations
+ */
+ @SuppressWarnings("unchecked")
+ protected Optional<ProcessResult> processMessage(String sensorType, JSONObject message, RawMessage rawMessage,
+ MessageParser<JSONObject> parser,
+ ParserConfigurations parserConfigurations
+ ) {
+ Optional<ProcessResult> processResult = Optional.empty();
+ SensorParserConfig sensorParserConfig = parserConfigurations.getSensorParserConfig(sensorType);
+ sensorParserConfig.getRawMessageStrategy().mergeMetadata(
+ message,
+ rawMessage.getMetadata(),
+ sensorParserConfig.getMergeMetadata(),
+ sensorParserConfig.getRawMessageStrategyConfig()
+ );
+ message.put(Constants.SENSOR_TYPE, sensorType);
+ applyFieldTransformations(message, rawMessage, sensorParserConfig);
+ if (!message.containsKey(Constants.GUID)) {
+ message.put(Constants.GUID, UUID.randomUUID().toString());
+ }
+ MessageFilter<JSONObject> filter = sensorToParserComponentMap.get(sensorType).getFilter();
+ if (filter == null || filter.emit(message, stellarContext)) {
+ boolean isInvalid = !parser.validate(message);
+ List<FieldValidator> failedValidators = null;
+ if (!isInvalid) {
+ failedValidators = getFailedValidators(message, parserConfigurations);
+ isInvalid = !failedValidators.isEmpty();
+ }
+ if (isInvalid) {
+ MetronError error = new MetronError()
+ .withErrorType(Constants.ErrorType.PARSER_INVALID)
+ .withSensorType(Collections.singleton(sensorType))
+ .addRawMessage(message);
+ Set<String> errorFields = failedValidators == null ? null : failedValidators.stream()
+ .flatMap(fieldValidator -> fieldValidator.getInput().stream())
+ .collect(Collectors.toSet());
+ if (errorFields != null && !errorFields.isEmpty()) {
+ error.withErrorFields(errorFields);
+ }
+ processResult = Optional.of(new ProcessResult(error));
+ } else {
+ processResult = Optional.of(new ProcessResult(message));
+ }
+ }
+ return processResult;
+ }
+
+ /**
+ * Applies Stellar field transformations defined in the sensor parser config.
+ * @param message Message parsed by the MessageParser
+ * @param rawMessage Raw message including metadata
+ * @param sensorParserConfig Sensor parser config
+ */
+ private void applyFieldTransformations(JSONObject message, RawMessage rawMessage, SensorParserConfig sensorParserConfig) {
+ for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) {
+ if (handler != null) {
+ if (!sensorParserConfig.getMergeMetadata()) {
+ //if we haven't merged metadata, then we need to pass them along as configuration params.
+ handler.transformAndUpdate(
+ message,
+ stellarContext,
+ sensorParserConfig.getParserConfig(),
+ rawMessage.getMetadata()
+ );
+ } else {
+ handler.transformAndUpdate(
+ message,
+ stellarContext,
+ sensorParserConfig.getParserConfig()
+ );
+ }
+ }
+ }
+ }
+
+ private List<FieldValidator> getFailedValidators(JSONObject message, ParserConfigurations parserConfigurations) {
+ List<FieldValidator> fieldValidations = parserConfigurations.getFieldValidations();
+ List<FieldValidator> failedValidators = new ArrayList<>();
+ for(FieldValidator validator : fieldValidations) {
+ if(!validator.isValid(message, parserConfigurations.getGlobalConfig(), stellarContext)) {
+ failedValidators.add(validator);
+ }
+ }
+ return failedValidators;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunnerResults.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunnerResults.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunnerResults.java
new file mode 100644
index 0000000..7ca853c
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ParserRunnerResults.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers;
+
+import org.apache.metron.common.error.MetronError;
+
+import java.util.List;
+
+/**
+ * Container for the results of parsing a message with a ParserRunner.
+ * @param <T> The type of a successfully parsed message.
+ */
+public interface ParserRunnerResults<T> {
+
+ List<T> getMessages();
+
+ List<MetronError> getErrors();
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
index 05334c2..a9ee305 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
@@ -18,13 +18,18 @@
package org.apache.metron.parsers.bolt;
+import java.io.Serializable;
+import java.lang.invoke.MethodHandles;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.function.Function;
import com.github.benmanes.caffeine.cache.Cache;
-import org.apache.commons.lang3.StringUtils;
import org.apache.metron.common.Constants;
import org.apache.metron.common.bolt.ConfiguredParserBolt;
-import org.apache.metron.common.configuration.FieldTransformer;
-import org.apache.metron.common.configuration.FieldValidator;
+import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.error.MetronError;
@@ -33,12 +38,8 @@ import org.apache.metron.common.message.MessageGetters;
import org.apache.metron.common.message.metadata.RawMessage;
import org.apache.metron.common.message.metadata.RawMessageUtil;
import org.apache.metron.common.utils.ErrorUtils;
-import org.apache.metron.parsers.filters.Filters;
-import org.apache.metron.parsers.interfaces.MessageFilter;
-import org.apache.metron.parsers.interfaces.MessageParser;
-import org.apache.metron.parsers.interfaces.MessageParserResult;
-import org.apache.metron.parsers.interfaces.MultilineMessageParser;
-import org.apache.metron.parsers.topology.ParserComponents;
+import org.apache.metron.parsers.ParserRunner;
+import org.apache.metron.parsers.ParserRunnerResults;
import org.apache.metron.stellar.common.CachingStellarProcessor;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.dsl.StellarFunctions;
@@ -56,45 +57,32 @@ import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.Serializable;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Optional;
-import java.util.Set;
-import java.util.UUID;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
public class ParserBolt extends ConfiguredParserBolt implements Serializable {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private OutputCollector collector;
- private Map<String, ParserComponents> sensorToComponentMap;
+ private ParserRunner<JSONObject> parserRunner;
+ private Map<String, WriterHandler> sensorToWriterMap;
private Map<String, String> topicToSensorMap = new HashMap<>();
- private Context stellarContext;
private transient MessageGetStrategy messageGetStrategy;
- private transient Cache<CachingStellarProcessor.Key, Object> cache;
private int requestedTickFreqSecs;
private int defaultBatchTimeout;
private int batchTimeoutDivisor = 1;
public ParserBolt( String zookeeperUrl
- , Map<String, ParserComponents> sensorToComponentMap
+ , ParserRunner parserRunner
+ , Map<String, WriterHandler> sensorToWriterMap
) {
super(zookeeperUrl);
- this.sensorToComponentMap = sensorToComponentMap;
+ this.parserRunner = parserRunner;
+ this.sensorToWriterMap = sensorToWriterMap;
// Ensure that all sensors are either bulk sensors or not bulk sensors. Can't mix and match.
Boolean handleAcks = null;
- for (Map.Entry<String, ParserComponents> entry : sensorToComponentMap.entrySet()) {
- boolean writerHandleAck = entry.getValue().getWriter().handleAck();
+ for (Map.Entry<String, WriterHandler> entry : sensorToWriterMap.entrySet()) {
+ boolean writerHandleAck = entry.getValue().handleAck();
if (handleAcks == null) {
handleAcks = writerHandleAck;
} else if (!handleAcks.equals(writerHandleAck)) {
@@ -130,21 +118,44 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable {
/**
* Used only for unit testing
- * @param defaultBatchTimeout
*/
- protected void setDefaultBatchTimeout(int defaultBatchTimeout) {
- this.defaultBatchTimeout = defaultBatchTimeout;
+ public int getBatchTimeoutDivisor() {
+ return batchTimeoutDivisor;
}
/**
* Used only for unit testing
*/
- public int getDefaultBatchTimeout() {
- return defaultBatchTimeout;
+ protected void setSensorToWriterMap(Map<String, WriterHandler> sensorToWriterMap) {
+ this.sensorToWriterMap = sensorToWriterMap;
}
- public Map<String, ParserComponents> getSensorToComponentMap() {
- return sensorToComponentMap;
+ /**
+ * Used only for unit testing
+ */
+ protected Map<String, String> getTopicToSensorMap() {
+ return topicToSensorMap;
+ }
+
+ /**
+ * Used only for unit testing
+ */
+ protected void setTopicToSensorMap(Map<String, String> topicToSensorMap) {
+ this.topicToSensorMap = topicToSensorMap;
+ }
+
+ /**
+ * Used only for unit testing
+ */
+ public void setMessageGetStrategy(MessageGetStrategy messageGetStrategy) {
+ this.messageGetStrategy = messageGetStrategy;
+ }
+
+ /**
+ * Used only for unit testing
+ */
+ public void setOutputCollector(OutputCollector collector) {
+ this.collector = collector;
}
/**
@@ -159,7 +170,7 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable {
// to get the valid WriterConfiguration. But don't store any non-serializable objects,
// else Storm will throw a runtime error.
Function<WriterConfiguration, WriterConfiguration> configurationXform;
- WriterHandler writer = sensorToComponentMap.entrySet().iterator().next().getValue().getWriter();
+ WriterHandler writer = sensorToWriterMap.entrySet().iterator().next().getValue();
if (writer.isWriterToBulkWriter()) {
configurationXform = WriterToBulkWriter.TRANSFORMATION;
} else {
@@ -189,37 +200,11 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable {
super.prepare(stormConf, context, collector);
messageGetStrategy = MessageGetters.DEFAULT_BYTES_FROM_POSITION.get();
this.collector = collector;
-
- // Build the Stellar cache
- Map<String, Object> cacheConfig = new HashMap<>();
- for (Map.Entry<String, ParserComponents> entry: sensorToComponentMap.entrySet()) {
- String sensor = entry.getKey();
- SensorParserConfig config = getSensorParserConfig(sensor);
-
- if (config != null) {
- cacheConfig.putAll(config.getCacheConfig());
- }
- }
- cache = CachingStellarProcessor.createCache(cacheConfig);
+ this.parserRunner.init(this::getConfigurations, initializeStellar());
// Need to prep all sensors
- for (Map.Entry<String, ParserComponents> entry: sensorToComponentMap.entrySet()) {
+ for (Map.Entry<String, WriterHandler> entry: sensorToWriterMap.entrySet()) {
String sensor = entry.getKey();
- MessageParser<JSONObject> parser = entry.getValue().getMessageParser();
-
- initializeStellar();
- if (getSensorParserConfig(sensor) != null && sensorToComponentMap.get(sensor).getFilter() == null) {
- getSensorParserConfig(sensor).getParserConfig().putIfAbsent("stellarContext", stellarContext);
- if (!StringUtils.isEmpty(getSensorParserConfig(sensor).getFilterClassName())) {
- MessageFilter<JSONObject> filter = Filters.get(
- getSensorParserConfig(sensor).getFilterClassName(),
- getSensorParserConfig(sensor).getParserConfig()
- );
- getSensorToComponentMap().get(sensor).setFilter(filter);
- }
- }
-
- parser.init();
SensorParserConfig config = getSensorParserConfig(sensor);
if (config != null) {
@@ -229,9 +214,8 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable {
throw new IllegalStateException(
"Unable to retrieve a parser config for " + sensor);
}
- parser.configure(config.getParserConfig());
- WriterHandler writer = sensorToComponentMap.get(sensor).getWriter();
+ WriterHandler writer = sensorToWriterMap.get(sensor);
writer.init(stormConf, context, collector, getConfigurations());
if (defaultBatchTimeout == 0) {
//This means getComponentConfiguration was never called to initialize defaultBatchTimeout,
@@ -246,225 +230,106 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable {
}
}
- protected void initializeStellar() {
- Context.Builder builder = new Context.Builder()
- .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client)
- .with(Context.Capabilities.GLOBAL_CONFIG, () -> getConfigurations().getGlobalConfig())
- .with(Context.Capabilities.STELLAR_CONFIG, () -> getConfigurations().getGlobalConfig())
- ;
- if(cache != null) {
- builder = builder.with(Context.Capabilities.CACHE, () -> cache);
- }
- this.stellarContext = builder.build();
- StellarFunctions.initialize(stellarContext);
- }
-
@SuppressWarnings("unchecked")
@Override
public void execute(Tuple tuple) {
if (TupleUtils.isTick(tuple)) {
- try {
- for (Entry<String, ParserComponents> entry : sensorToComponentMap.entrySet()) {
- entry.getValue().getWriter().flush(getConfigurations(), messageGetStrategy);
- }
- } catch (Exception e) {
- throw new RuntimeException(
- "This should have been caught in the writerHandler. If you see this, file a JIRA", e);
- } finally {
- collector.ack(tuple);
- }
+ handleTickTuple(tuple);
return;
}
-
byte[] originalMessage = (byte[]) messageGetStrategy.get(tuple);
+ String topic = tuple.getStringByField(FieldsConfiguration.TOPIC.getFieldName());
+ String sensorType = topicToSensorMap.get(topic);
try {
- SensorParserConfig sensorParserConfig;
- MessageParser<JSONObject> parser;
- String sensor;
- Map<String, Object> metadata;
- if (sensorToComponentMap.size() == 1) {
- // There's only one parser, so grab info directly
- Entry<String, ParserComponents> sensorParser = sensorToComponentMap.entrySet().iterator()
- .next();
- sensor = sensorParser.getKey();
- parser = sensorParser.getValue().getMessageParser();
- sensorParserConfig = getSensorParserConfig(sensor);
- } else {
- // There's multiple parsers, so pull the topic from the Tuple and look up the sensor
- String topic = tuple.getStringByField(FieldsConfiguration.TOPIC.getFieldName());
- sensor = topicToSensorMap.get(topic);
- parser = sensorToComponentMap.get(sensor).getMessageParser();
- sensorParserConfig = getSensorParserConfig(sensor);
- }
+ ParserConfigurations parserConfigurations = getConfigurations();
+ SensorParserConfig sensorParserConfig = parserConfigurations.getSensorParserConfig(sensorType);
+ RawMessage rawMessage = RawMessageUtil.INSTANCE.getRawMessage( sensorParserConfig.getRawMessageStrategy()
+ , tuple
+ , originalMessage
+ , sensorParserConfig.getReadMetadata()
+ , sensorParserConfig.getRawMessageStrategyConfig()
+ );
+ ParserRunnerResults<JSONObject> parserRunnerResults = parserRunner.execute(sensorType, rawMessage, parserConfigurations);
+ long numWritten = parserRunnerResults.getMessages().stream()
+ .map(message -> handleMessage(sensorType, originalMessage, tuple, message, collector))
+ .filter(result -> result)
+ .count();
+ parserRunnerResults.getErrors().forEach(error -> ErrorUtils.handleError(collector, error));
- List<FieldValidator> fieldValidations = getConfigurations().getFieldValidations();
- boolean ackTuple = false;
- int numWritten = 0;
- if (sensorParserConfig != null) {
- RawMessage rawMessage = RawMessageUtil.INSTANCE.getRawMessage( sensorParserConfig.getRawMessageStrategy()
- , tuple
- , originalMessage
- , sensorParserConfig.getReadMetadata()
- , sensorParserConfig.getRawMessageStrategyConfig()
- );
-
- metadata = rawMessage.getMetadata();
-
- MultilineMessageParser mmp = null;
- if (!(parser instanceof MultilineMessageParser)) {
- mmp = new MultilineMessageParser<JSONObject>() {
-
- @Override
- public void configure(Map<String, Object> config) {
- parser.configure(config);
- }
-
- @Override
- public void init() {
- parser.init();
- }
-
- @Override
- public boolean validate(JSONObject message) {
- return parser.validate(message);
- }
-
- @Override
- public List<JSONObject> parse(byte[] message) {
- return parser.parse(message);
- }
-
- @Override
- public Optional<List<JSONObject>> parseOptional(byte[] message) {
- return parser.parseOptional(message);
- }
- };
- } else {
- mmp = (MultilineMessageParser) parser;
- }
-
- Optional<MessageParserResult<JSONObject>> results = mmp.parseOptionalResult(rawMessage.getMessage());
-
- // check if there is a master error
- if (results.isPresent() && results.get().getMasterThrowable().isPresent()) {
- handleError(originalMessage, tuple, results.get().getMasterThrowable().get(), collector);
- return;
- }
-
- // Handle the message results
- List<JSONObject> messages = results.isPresent() ? results.get().getMessages() : Collections.EMPTY_LIST;
- for (JSONObject message : messages) {
- //we want to ack the tuple in the situation where we have are not doing a bulk write
- //otherwise we want to defer to the writerComponent who will ack on bulk commit.
- WriterHandler writer = sensorToComponentMap.get(sensor).getWriter();
- ackTuple = !writer.handleAck();
-
- sensorParserConfig.getRawMessageStrategy().mergeMetadata(
- message,
- metadata,
- sensorParserConfig.getMergeMetadata(),
- sensorParserConfig.getRawMessageStrategyConfig()
- );
- message.put(Constants.SENSOR_TYPE, sensor);
-
- for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) {
- if (handler != null) {
- if (!sensorParserConfig.getMergeMetadata()) {
- //if we haven't merged metadata, then we need to pass them along as configuration params.
- handler.transformAndUpdate(
- message,
- stellarContext,
- sensorParserConfig.getParserConfig(),
- metadata
- );
- } else {
- handler.transformAndUpdate(
- message,
- stellarContext,
- sensorParserConfig.getParserConfig()
- );
- }
- }
- }
- if (!message.containsKey(Constants.GUID)) {
- message.put(Constants.GUID, UUID.randomUUID().toString());
- }
-
- MessageFilter<JSONObject> filter = sensorToComponentMap.get(sensor).getFilter();
- if (filter == null || filter.emitTuple(message, stellarContext)) {
- boolean isInvalid = !parser.validate(message);
- List<FieldValidator> failedValidators = null;
- if (!isInvalid) {
- failedValidators = getFailedValidators(message, fieldValidations);
- isInvalid = !failedValidators.isEmpty();
- }
- if (isInvalid) {
- MetronError error = new MetronError()
- .withErrorType(Constants.ErrorType.PARSER_INVALID)
- .withSensorType(Collections.singleton(sensor))
- .addRawMessage(message);
- Set<String> errorFields = failedValidators == null ? null : failedValidators.stream()
- .flatMap(fieldValidator -> fieldValidator.getInput().stream())
- .collect(Collectors.toSet());
- if (errorFields != null && !errorFields.isEmpty()) {
- error.withErrorFields(errorFields);
- }
- ErrorUtils.handleError(collector, error);
- } else {
- numWritten++;
- writer.write(sensor, tuple, message, getConfigurations(), messageGetStrategy);
- }
- }
- }
-
- // Handle the error results
- Map<Object, Throwable> messageErrors = results.isPresent()
- ? results.get().getMessageThrowables() : Collections.EMPTY_MAP;
-
- for (Entry<Object,Throwable> entry : messageErrors.entrySet()) {
- MetronError error = new MetronError()
- .withErrorType(Constants.ErrorType.PARSER_ERROR)
- .withThrowable(entry.getValue())
- .withSensorType(sensorToComponentMap.keySet())
- .addRawMessage(originalMessage);
- ErrorUtils.handleError(collector, error);
- }
- }
//if we are supposed to ack the tuple OR if we've never passed this tuple to the bulk writer
//(meaning that none of the messages are valid either globally or locally)
//then we want to handle the ack ourselves.
- if (ackTuple || numWritten == 0) {
+ if (!sensorToWriterMap.get(sensorType).handleAck() || numWritten == 0) {
collector.ack(tuple);
}
} catch (Throwable ex) {
- handleError(originalMessage, tuple, ex, collector);
+ handleError(sensorType, originalMessage, tuple, ex, collector);
+ collector.ack(tuple);
}
}
- protected void handleError(byte[] originalMessage, Tuple tuple, Throwable ex, OutputCollector collector) {
+ protected Context initializeStellar() {
+ Map<String, Object> cacheConfig = new HashMap<>();
+ for (String sensorType: this.parserRunner.getSensorTypes()) {
+ SensorParserConfig config = getSensorParserConfig(sensorType);
+
+ if (config != null) {
+ cacheConfig.putAll(config.getCacheConfig());
+ }
+ }
+ Cache<CachingStellarProcessor.Key, Object> cache = CachingStellarProcessor.createCache(cacheConfig);
+
+ Context.Builder builder = new Context.Builder()
+ .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client)
+ .with(Context.Capabilities.GLOBAL_CONFIG, () -> getConfigurations().getGlobalConfig())
+ .with(Context.Capabilities.STELLAR_CONFIG, () -> getConfigurations().getGlobalConfig())
+ ;
+ if(cache != null) {
+ builder = builder.with(Context.Capabilities.CACHE, () -> cache);
+ }
+ Context stellarContext = builder.build();
+ StellarFunctions.initialize(stellarContext);
+ return stellarContext;
+ }
+
+ protected void handleTickTuple(Tuple tuple) {
+ try {
+ for (Entry<String, WriterHandler> entry : sensorToWriterMap.entrySet()) {
+ entry.getValue().flush(getConfigurations(), messageGetStrategy);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "This should have been caught in the writerHandler. If you see this, file a JIRA", e);
+ } finally {
+ collector.ack(tuple);
+ }
+ }
+
+ protected boolean handleMessage(String sensorType, byte[] originalMessage, Tuple tuple, JSONObject message, OutputCollector collector) {
+ WriterHandler writer = sensorToWriterMap.get(sensorType);
+ try {
+ writer.write(sensorType, tuple, message, getConfigurations(), messageGetStrategy);
+ return true;
+ } catch (Exception ex) {
+ handleError(sensorType, originalMessage, tuple, ex, collector);
+ return false;
+ }
+ }
+
+ protected void handleError(String sensorType, byte[] originalMessage, Tuple tuple, Throwable ex, OutputCollector collector) {
MetronError error = new MetronError()
.withErrorType(Constants.ErrorType.PARSER_ERROR)
.withThrowable(ex)
- .withSensorType(sensorToComponentMap.keySet())
+ .withSensorType(Collections.singleton(sensorType))
.addRawMessage(originalMessage);
ErrorUtils.handleError(collector, error);
- collector.ack(tuple);
- }
-
- private List<FieldValidator> getFailedValidators(JSONObject input, List<FieldValidator> validators) {
- List<FieldValidator> failedValidators = new ArrayList<>();
- for(FieldValidator validator : validators) {
- if(!validator.isValid(input, getConfigurations().getGlobalConfig(), stellarContext)) {
- failedValidators.add(validator);
- }
- }
- return failedValidators;
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declareStream(Constants.ERROR_STREAM, new Fields("message"));
}
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java
index 9cdafa3..1fa1feb 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/BroMessageFilter.java
@@ -67,7 +67,7 @@ public class BroMessageFilter implements MessageFilter<JSONObject>{
*/
@Override
- public boolean emitTuple(JSONObject message, Context context) {
+ public boolean emit(JSONObject message, Context context) {
String protocol = (String) message.get(_key);
return _known_protocols.contains(protocol);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java
index 15a035a..8300ff4 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/filters/StellarFilter.java
@@ -54,7 +54,7 @@ public class StellarFilter implements MessageFilter<JSONObject> {
}
@Override
- public boolean emitTuple(JSONObject message, Context context) {
+ public boolean emit(JSONObject message, Context context) {
VariableResolver resolver = new MapVariableResolver(message);
return processor.parse(query, resolver, functionResolver, context);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java
index b7b91c0..207c070 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageFilter.java
@@ -21,5 +21,5 @@ import org.apache.metron.stellar.dsl.Context;
public interface MessageFilter<T> extends Configurable{
- boolean emitTuple(T message, Context context);
+ boolean emit(T message, Context context);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
index 665076b..c9f8351 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.parsers.interfaces;
+import org.apache.commons.lang3.NotImplementedException;
import org.apache.metron.parsers.DefaultMessageParserResult;
import java.io.Serializable;
@@ -38,18 +39,42 @@ public interface MessageParser<T> extends Configurable {
* @param rawMessage the raw bytes of the message
* @return If null is returned, this is treated as an empty list.
*/
- List<T> parse(byte[] rawMessage);
+ @Deprecated
+ default List<T> parse(byte[] rawMessage) {
+ throw new NotImplementedException("parse is not implemented");
+ }
/**
* Take raw data and convert it to an optional list of messages.
* @param parseMessage the raw bytes of the message
* @return If null is returned, this is treated as an empty list.
*/
+ @Deprecated
default Optional<List<T>> parseOptional(byte[] parseMessage) {
return Optional.ofNullable(parse(parseMessage));
}
/**
+ * Take raw data and convert it to messages. Each raw message may produce multiple messages and therefore
+ * multiple errors. A {@link MessageParserResult} is returned, which will have both the messages produced
+ * and the errors.
+ * @param parseMessage the raw bytes of the message
+ * @return Optional of {@link MessageParserResult}
+ */
+ default Optional<MessageParserResult<T>> parseOptionalResult(byte[] parseMessage) {
+ Optional<MessageParserResult<T>> result = Optional.empty();
+ try {
+ Optional<List<T>> optionalMessages = parseOptional(parseMessage);
+ if (optionalMessages.isPresent()) {
+ result = Optional.of(new DefaultMessageParserResult<>(optionalMessages.get()));
+ }
+ } catch (Throwable t) {
+ return Optional.of(new DefaultMessageParserResult<>(t));
+ }
+ return result;
+ }
+
+ /**
* Validate the message to ensure that it's correct.
* @param message the message to validate
* @return true if the message is valid, false if not
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MultilineMessageParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MultilineMessageParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MultilineMessageParser.java
deleted file mode 100644
index 7818f9a..0000000
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MultilineMessageParser.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.parsers.interfaces;
-
-import org.apache.commons.lang3.NotImplementedException;
-import org.apache.metron.parsers.DefaultMessageParserResult;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-
-public interface MultilineMessageParser<T> extends MessageParser<T> {
-
- default List<T> parse(byte[] rawMessage) {
- throw new NotImplementedException("parse is not implemented");
- }
-
- /**
- * Take raw data and convert it to messages. Each raw message may produce multiple messages and therefore
- * multiple errors. A {@link MessageParserResult} is returned, which will have both the messages produced
- * and the errors.
- * @param parseMessage the raw bytes of the message
- * @return Optional of {@link MessageParserResult}
- */
- default Optional<MessageParserResult<T>> parseOptionalResult(byte[] parseMessage) {
- List<T> list = new ArrayList<>();
- try {
- Optional<List<T>> optionalMessages = parseOptional(parseMessage);
- optionalMessages.ifPresent(list::addAll);
- } catch (Throwable t) {
- return Optional.of(new DefaultMessageParserResult<>(t));
- }
- return Optional.of(new DefaultMessageParserResult<T>(list));
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
index 79a082a..5b62e85 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
@@ -26,8 +26,8 @@ import com.github.palindromicity.syslog.dsl.SyslogFieldKeys;
import org.apache.commons.lang3.StringUtils;
import org.apache.metron.parsers.BasicParser;
import org.apache.metron.parsers.DefaultMessageParserResult;
+import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.parsers.interfaces.MessageParserResult;
-import org.apache.metron.parsers.interfaces.MultilineMessageParser;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,7 +52,7 @@ import java.util.Optional;
/**
* Parser for well structured RFC 5424 messages.
*/
-public class Syslog5424Parser implements MultilineMessageParser<JSONObject>, Serializable {
+public class Syslog5424Parser implements MessageParser<JSONObject>, Serializable {
protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String NIL_POLICY_CONFIG = "nilPolicy";
private transient SyslogParser syslogParser;
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserComponent.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserComponent.java
new file mode 100644
index 0000000..eb5ff9f
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserComponent.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.topology;
+
+import org.apache.metron.parsers.interfaces.MessageFilter;
+import org.apache.metron.parsers.interfaces.MessageParser;
+import org.json.simple.JSONObject;
+
+import java.io.Serializable;
+
+public class ParserComponent implements Serializable {
+ private static final long serialVersionUID = 7880346740026374665L;
+
+ private MessageParser<JSONObject> messageParser;
+ private MessageFilter<JSONObject> filter;
+
+ public ParserComponent(
+ MessageParser<JSONObject> messageParser,
+ MessageFilter<JSONObject> filter) {
+ this.messageParser = messageParser;
+ this.filter = filter;
+ }
+
+ public MessageParser<JSONObject> getMessageParser() {
+ return messageParser;
+ }
+
+ public MessageFilter<JSONObject> getFilter() {
+ return filter;
+ }
+
+ public void setMessageParser(
+ MessageParser<JSONObject> messageParser) {
+ this.messageParser = messageParser;
+ }
+
+ public void setFilter(
+ MessageFilter<JSONObject> filter) {
+ this.filter = filter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserComponents.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserComponents.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserComponents.java
deleted file mode 100644
index 32d56b9..0000000
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserComponents.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers.topology;
-
-import java.io.Serializable;
-import org.apache.metron.parsers.bolt.WriterHandler;
-import org.apache.metron.parsers.interfaces.MessageFilter;
-import org.apache.metron.parsers.interfaces.MessageParser;
-import org.json.simple.JSONObject;
-
-public class ParserComponents implements Serializable {
- private static final long serialVersionUID = 7880346740026374665L;
-
- private MessageParser<JSONObject> messageParser;
- private MessageFilter<JSONObject> filter;
- private WriterHandler writer;
-
- public ParserComponents(
- MessageParser<JSONObject> messageParser,
- MessageFilter<JSONObject> filter,
- WriterHandler writer) {
- this.messageParser = messageParser;
- this.filter = filter;
- this.writer = writer;
- }
-
- public MessageParser<JSONObject> getMessageParser() {
- return messageParser;
- }
-
- public MessageFilter<JSONObject> getFilter() {
- return filter;
- }
-
- public WriterHandler getWriter() {
- return writer;
- }
-
- public void setMessageParser(
- MessageParser<JSONObject> messageParser) {
- this.messageParser = messageParser;
- }
-
- public void setFilter(
- MessageFilter<JSONObject> filter) {
- this.filter = filter;
- }
-
- public void setWriter(WriterHandler writer) {
- this.writer = writer;
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
index d20e1a5..9dc7b88 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,11 +21,11 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
-import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.metron.common.Constants;
@@ -37,12 +37,10 @@ import org.apache.metron.common.utils.KafkaUtils;
import org.apache.metron.common.utils.ReflectionUtils;
import org.apache.metron.common.writer.BulkMessageWriter;
import org.apache.metron.common.writer.MessageWriter;
+import org.apache.metron.parsers.ParserRunnerImpl;
import org.apache.metron.parsers.bolt.ParserBolt;
import org.apache.metron.parsers.bolt.WriterBolt;
import org.apache.metron.parsers.bolt.WriterHandler;
-import org.apache.metron.parsers.filters.Filters;
-import org.apache.metron.parsers.interfaces.MessageFilter;
-import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.parsers.topology.config.ValueSupplier;
import org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder;
import org.apache.metron.storm.kafka.flux.SpoutConfiguration;
@@ -268,29 +266,14 @@ public class ParserTopologyBuilder {
Optional<String> securityProtocol,
ParserConfigurations configs,
Optional<String> outputTopic) {
-
- Map<String, ParserComponents> parserBoltConfigs = new HashMap<>();
+ Map<String, WriterHandler> writerConfigs = new HashMap<>();
for( Entry<String, SensorParserConfig> entry : sensorTypeToParserConfig.entrySet()) {
String sensorType = entry.getKey();
SensorParserConfig parserConfig = entry.getValue();
- // create message parser
- MessageParser<JSONObject> parser = ReflectionUtils
- .createInstance(parserConfig.getParserClassName());
- parser.configure(parserConfig.getParserConfig());
-
- // create message filter
- MessageFilter<JSONObject> filter = null;
- if (!StringUtils.isEmpty(parserConfig.getFilterClassName())) {
- filter = Filters.get(
- parserConfig.getFilterClassName(),
- parserConfig.getParserConfig()
- );
- }
// create a writer
AbstractWriter writer;
if (parserConfig.getWriterClassName() == null) {
-
// if not configured, use a sensible default
writer = createKafkaWriter(brokerUrl, zookeeperUrl, securityProtocol)
.withTopic(outputTopic.orElse(Constants.ENRICHMENT_TOPIC));
@@ -304,16 +287,10 @@ public class ParserTopologyBuilder {
// create a writer handler
WriterHandler writerHandler = createWriterHandler(writer);
-
- ParserComponents components = new ParserComponents(
- parser,
- filter,
- writerHandler
- );
- parserBoltConfigs.put(sensorType, components);
+ writerConfigs.put(sensorType, writerHandler);
}
- return new ParserBolt(zookeeperUrl, parserBoltConfigs);
+ return new ParserBolt(zookeeperUrl, new ParserRunnerImpl(new HashSet<>(sensorTypeToParserConfig.keySet())), writerConfigs);
}
/**
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java
index 8441409..2f3784a 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/filters/FiltersTest.java
@@ -42,8 +42,8 @@ public class FiltersTest {
put("filter.query", "exists(foo)");
}};
MessageFilter<JSONObject> filter = Filters.get(Filters.STELLAR.name(), config);
- Assert.assertTrue(filter.emitTuple(new JSONObject(ImmutableMap.of("foo", 1)), Context.EMPTY_CONTEXT()));
- Assert.assertFalse(filter.emitTuple(new JSONObject(ImmutableMap.of("bar", 1)), Context.EMPTY_CONTEXT()));
+ Assert.assertTrue(filter.emit(new JSONObject(ImmutableMap.of("foo", 1)), Context.EMPTY_CONTEXT()));
+ Assert.assertFalse(filter.emit(new JSONObject(ImmutableMap.of("bar", 1)), Context.EMPTY_CONTEXT()));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java
index 9769baa..4842a1f 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MessageParserTest.java
@@ -19,68 +19,118 @@
package org.apache.metron.parsers;
import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.json.simple.JSONObject;
import org.junit.Assert;
import org.junit.Test;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
public class MessageParserTest {
- @Test
- public void testNullable() throws Exception {
- MessageParser parser = new MessageParser() {
- @Override
- public void init() {
- }
+ abstract class TestMessageParser implements MessageParser<JSONObject> {
+ @Override
+ public void init() {
+ }
- @Override
- public List parse(byte[] rawMessage) {
- return null;
- }
+ @Override
+ public boolean validate(JSONObject message) {
+ return false;
+ }
- @Override
- public boolean validate(Object message) {
- return false;
- }
+ @Override
+ public void configure(Map<String, Object> config) {
- @Override
- public void configure(Map<String, Object> config) {
+ }
+ }
+ @Test
+ public void testNullable() throws Exception {
+ MessageParser parser = new TestMessageParser() {
+ @Override
+ public List<JSONObject> parse(byte[] rawMessage) {
+ return null;
}
};
- Assert.assertNotNull(parser.parseOptional(null));
- Assert.assertFalse(parser.parseOptional(null).isPresent());
+ Assert.assertNotNull(parser.parseOptionalResult(null));
+ Assert.assertFalse(parser.parseOptionalResult(null).isPresent());
}
@Test
public void testNotNullable() throws Exception {
- MessageParser parser = new MessageParser() {
+ MessageParser<JSONObject> parser = new TestMessageParser() {
@Override
- public void init() {
-
+ public List<JSONObject> parse(byte[] rawMessage) {
+ return new ArrayList<>();
}
+ };
+ Assert.assertNotNull(parser.parseOptionalResult(null));
+ Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult(null);
+ Assert.assertTrue(ret.isPresent());
+ Assert.assertEquals(0, ret.get().getMessages().size());
+ }
+ @Test
+ public void testParse() {
+ JSONObject message = new JSONObject();
+ MessageParser<JSONObject> parser = new TestMessageParser() {
@Override
- public List parse(byte[] rawMessage) {
- return new ArrayList<>();
+ public List<JSONObject> parse(byte[] rawMessage) {
+ return Collections.singletonList(message);
}
+ };
+ Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes());
+ Assert.assertTrue(ret.isPresent());
+ Assert.assertEquals(1, ret.get().getMessages().size());
+ Assert.assertEquals(message, ret.get().getMessages().get(0));
+ }
+ @Test
+ public void testParseOptional() {
+ JSONObject message = new JSONObject();
+ MessageParser<JSONObject> parser = new TestMessageParser() {
@Override
- public boolean validate(Object message) {
- return false;
+ public Optional<List<JSONObject>> parseOptional(byte[] rawMessage) {
+ return Optional.of(Collections.singletonList(message));
}
+ };
+ Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes());
+ Assert.assertTrue(ret.isPresent());
+ Assert.assertEquals(1, ret.get().getMessages().size());
+ Assert.assertEquals(message, ret.get().getMessages().get(0));
+ }
+ @Test
+ public void testParseException() {
+ MessageParser<JSONObject> parser = new TestMessageParser() {
@Override
- public void configure(Map<String, Object> config) {
+ public List<JSONObject> parse(byte[] rawMessage) {
+ throw new RuntimeException("parse exception");
+ }
+ };
+ Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes());
+ Assert.assertTrue(ret.isPresent());
+ Assert.assertTrue(ret.get().getMasterThrowable().isPresent());
+ Assert.assertEquals("parse exception", ret.get().getMasterThrowable().get().getMessage());
+ }
+ @Test
+ public void testParseOptionalException() {
+ MessageParser<JSONObject> parser = new TestMessageParser() {
+ @Override
+ public Optional<List<JSONObject>> parseOptional(byte[] rawMessage) {
+ throw new RuntimeException("parse exception");
}
};
- Assert.assertNotNull(parser.parseOptional(null));
- Optional<List> ret = parser.parseOptional(null);
+ Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes());
Assert.assertTrue(ret.isPresent());
- Assert.assertEquals(0, ret.get().size());
+ Assert.assertTrue(ret.get().getMasterThrowable().isPresent());
+ Assert.assertEquals("parse exception", ret.get().getMasterThrowable().get().getMessage());
}
+
}
[37/51] [abbrv] metron git commit: METRON-1769 Script creation of a
release candidate (justinleet) closes apache/metron#1188
Posted by ot...@apache.org.
METRON-1769 Script creation of a release candidate (justinleet) closes apache/metron#1188
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f153375f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f153375f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f153375f
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: f153375fad65e75a10ce490b4a5f29f7e82f70dd
Parents: 3d923cd
Author: justinleet <ju...@gmail.com>
Authored: Wed Oct 10 15:41:22 2018 -0400
Committer: leet <le...@apache.org>
Committed: Wed Oct 10 15:41:22 2018 -0400
----------------------------------------------------------------------
dev-utilities/release-utils/README.md | 129 +++++++
dev-utilities/release-utils/metron-rc-check | 6 +-
.../release-utils/prepare-release-candidate | 337 +++++++++++++++++++
3 files changed, 469 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/f153375f/dev-utilities/release-utils/README.md
----------------------------------------------------------------------
diff --git a/dev-utilities/release-utils/README.md b/dev-utilities/release-utils/README.md
new file mode 100644
index 0000000..d1fd816
--- /dev/null
+++ b/dev-utilities/release-utils/README.md
@@ -0,0 +1,129 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Release Tools
+
+This project contains tools to assist Apache Metron project committers.
+
+## Prepare Release Candidate
+
+This script automates the process of creating a release candidate from `apache/metron` or `apache/metron-bro-plugin-kafka`. The script will prompt for various information necessary. Ensure your signing key is setup per [Release Signing](https://www.apache.org/dev/release-signing.html) and [Apache GnuPGP Instructions](https://www.apache.org/dev/openpgp.html#gnupg)
+
+When prompted the `[value in brackets]` is used by default. To accept the default, simply press `enter`. If you would like to change the default, type it in and hit `enter` when done.
+
+In the following example, enter the appropriate information
+
+1. Execute the script.
+
+ The first time the script is run, you will be prompted for additional information including your Apache username and Apache email. These values are persisted in `~/.metron-prepare-release-candidate`. Subsequent executions of the script will retrieve these values, rather than prompting you again for them.
+
+ ```
+ $ ./prepare-release-candidate
+ your apache userid []: leet
+ your apache email [leet@apache.org]:
+ ```
+
+1. Select a repository we're creating an RC for.
+
+ ```
+ [1] metron
+ [2] metron-bro-plugin-kafka
+ which repo? [1]: 1
+ ```
+
+1. Enter the current version number. This will be the base for the CHANGES file
+
+ ```
+ current version: 0.6.0
+ ```
+
+1. Enter the version being built.
+
+ ```
+ version being built: 0.6.1
+ ```
+
+1. Enter the current RC number
+
+ ```
+ release candidate number: 1
+ ```
+
+1. Enter the branch we're releasing from. In most cases, this will be master, but for maintenance releases it can be another branch.
+
+ ```
+ base revision branch or hash for release candidate [master]:
+ ```
+
+1. Enter the signing key id.
+
+ ```
+ signing key id in 8-byte format (e.g. BADDCAFEDEADBEEF):
+ ```
+
+1. Enter if this is a practice run. In a practice run, nothing is pushed to SVN, but everything is setup and built otherwise.
+
+ ```
+ do a live run (push to remote repositories?) [y/n]
+ ```
+
+1. Wait for all repos to be checked out to complete. There will be some additional work done, e.g. along with branch and tag creation. In a live run, you may be prompted for Git credentials to push a branch.
+
+ ```
+ Checking out repo: https://dist.apache.org/repos/dist/dev/metron
+ Checking out repo: dev
+ Checking out repo: https://dist.apache.org/repos/dist/release/metron
+ Checking out repo: release
+ Checking out git repo: https://git-wip-us.apache.org/repos/asf/metron.git
+ Cloning into '/Users/justinleet/tmp/metron-0.6.1/metron'...
+ remote: Counting objects: 46146, done.
+ remote: Compressing objects: 100% (15568/15568), done.
+ remote: Total 46146 (delta 21513), reused 43696 (delta 19489)
+ Receiving objects: 100% (46146/46146), 56.00 MiB | 1.04 MiB/s, done.
+ Resolving deltas: 100% (21513/21513), done.
+ Creating branch: Metron_0.6.1
+ Using git rev: master
+ Already on 'master'
+ Your branch is up to date with 'origin/master'.
+ Switched to a new branch 'Metron_0.6.1'
+ This is a practice run. Not running <git push --set-upstream origin Metron_0.6.1>
+ Creating tentative git tag <0.6.1-rc1>. Do not push this tag until RC is ready for community review.
+ Already on 'Metron_0.6.1'
+ Creating the RC tarball for tag apache-metron-0.6.1-rc1
+ Creating the SHA hash files
+ ```
+
+1. Provide the passphrase to `gpg` to sign the artifacts.
+
+ ```
+ Signing the release tarball
+ Copying release artifacts
+ ```
+
+1. Shortly afterwards the RC will be finalized. In a practice run, this will not be pushed back to SVN.
+ ```
+ Creating CHANGES file
+ Extracting LICENSE, NOTICE, and KEYS from tarball
+ x LICENSE
+ x NOTICE
+ This is a practice run. Not running the following commands:
+ <svn add 0.6.1-RC1>
+ <svn commit -m "Adding artifacts for metron 0.6.1-RC1">
+ ```
+
+At this point, all RC artifacts have been created. In a live run, these will have been pushed to the appropriate repositories and are ready for community review.
http://git-wip-us.apache.org/repos/asf/metron/blob/f153375f/dev-utilities/release-utils/metron-rc-check
----------------------------------------------------------------------
diff --git a/dev-utilities/release-utils/metron-rc-check b/dev-utilities/release-utils/metron-rc-check
index e3cc39d..143ba85 100755
--- a/dev-utilities/release-utils/metron-rc-check
+++ b/dev-utilities/release-utils/metron-rc-check
@@ -181,7 +181,7 @@ fi
if [ -n "$BRO" ]; then
- METRON_KAFKA_BRO_ASSEMBLY="$METRON_RC_DIST/apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz"
+ METRON_KAFKA_BRO_ASSEMBLY="$METRON_RC_DIST/apache-metron-bro-plugin-kafka_$BRO_VERSION-$RC.tar.gz"
METRON_KAFKA_BRO_ASSEMBLY_ASC="$METRON_KAFKA_BRO_ASSEMBLY.asc"
echo "Downloading $METRON_KAFKA_BRO_ASSEMBLY"
@@ -214,12 +214,12 @@ fi
if [ -n "$BRO" ]; then
echo "Verifying Bro Kafka Plugin Assembly"
- if ! gpg --verify ./"apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz.asc" "apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" ; then
+ if ! gpg --verify ./"apache-metron-bro-plugin-kafka_$BRO_VERSION-$RC.tar.gz.asc" "apache-metron-bro-plugin-kafka_$BRO_VERSION-$RC.tar.gz" ; then
echo "[ERROR] failed to verify Bro Kafka Plugin Assembly"
exit 1
fi
- if ! tar -xzf "apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" ; then
+ if ! tar -xzf "apache-metron-bro-plugin-kafka_$BRO_VERSION-$RC.tar.gz" ; then
echo "[ERROR] failed to unpack Bro Kafka Plugin Assembly"
exit 1
fi
http://git-wip-us.apache.org/repos/asf/metron/blob/f153375f/dev-utilities/release-utils/prepare-release-candidate
----------------------------------------------------------------------
diff --git a/dev-utilities/release-utils/prepare-release-candidate b/dev-utilities/release-utils/prepare-release-candidate
new file mode 100755
index 0000000..bedee9b
--- /dev/null
+++ b/dev-utilities/release-utils/prepare-release-candidate
@@ -0,0 +1,337 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# Creates and sets up the actual release artifacts into dev
+# See https://cwiki.apache.org/confluence/display/METRON/Release+Process
+
+
+set -eo pipefail
+
+# define constants
+# Git repos
+# To add a new submodule, add the repo name, upstream Git repo, and update the getrepo() selection function.
+# if versioning of the submodule isn't x.y.z format, retrieval of the git tag must also be adjusted.
+METRON_REPO_NAME="metron"
+BRO_PLUGIN_REPO_NAME="metron-bro-plugin-kafka"
+METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${METRON_REPO_NAME}.git"
+BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${BRO_PLUGIN_REPO_NAME}.git"
+
+DEV_REPO="https://dist.apache.org/repos/dist/dev/metron"
+RELEASE_REPO=" https://dist.apache.org/repos/dist/release/metron"
+PLUGIN_GIT_REPO="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git"
+
+RC_PREFIX=rc
+TAG_POSTFIX="-release"
+
+CONFIG_FILE=~/.metron-prepare-release-candidate
+# does a config file already exist?
+if [ -f $CONFIG_FILE ]; then
+ . $CONFIG_FILE
+ echo " ...using settings from $CONFIG_FILE"
+fi
+
+# apache id of committer (you)
+if [ -z "${APACHE_NAME}" ]; then
+ read -p " your apache userid [${APACHE_NAME}]: " INPUT
+ [ -n "$INPUT" ] && APACHE_NAME=$INPUT
+
+ # write setting to config file
+ echo "APACHE_NAME=$APACHE_NAME" >> $CONFIG_FILE
+fi
+
+# apache email addr of committer (you)
+if [ -z "${APACHE_EMAIL}" ]; then
+ APACHE_EMAIL=${APACHE_NAME}@apache.org
+ read -p " your apache email [${APACHE_EMAIL}]: " INPUT
+ [ -n "$INPUT" ] && APACHE_EMAIL=$INPUT
+
+ # write setting to config file, so it is not needed next time
+ echo "APACHE_EMAIL=$APACHE_EMAIL" >> $CONFIG_FILE
+fi
+
+# which repo? metron or metron-bro-plugin-kafka
+getrepo() {
+ echo " [1] ${METRON_REPO_NAME}"
+ echo " [2] ${BRO_PLUGIN_REPO_NAME}"
+ read -p " which repo? [1]: " INPUT
+ case "${INPUT}" in
+ [Bb][Rr][Oo]|[Mm][Ee][Tt][Rr][Oo][Nn]-[Bb][Rr][Oo]-[Pp][Ll][Uu][Gg][Ii][Nn]-[Kk][Aa][Ff][Kk][Aa]|*metron-bro-plugin-kafka\.git|2)
+ INPUT="${BRO_PLUGIN_UPSTREAM}" ;;
+ [Mm][Ee][Tt][Rr][Oo][Nn]|*metron\.git|1|'')
+ INPUT="${METRON_UPSTREAM}" ;;
+ *)
+ echo "Invalid repo, provided \"${INPUT}\". Please choose between ${METRON_REPO_NAME} or ${BRO_PLUGIN_REPO_NAME}"
+ return 1
+ ;;
+ esac
+ [ -n "$INPUT" ] && UPSTREAM=$INPUT
+ return 0
+}
+until getrepo; do :; done
+
+CHOSEN_REPO=$(basename ${UPSTREAM%%.git})
+# Need the capitalized version of the repos some naming
+CAPITAL_REPO="$(tr '[:lower:]' '[:upper:]' <<< ${CHOSEN_REPO:0:1})${CHOSEN_REPO:1}"
+
+getcurrentversion() {
+ # currently released version. Used for CHANGES file
+ read -p " current version: " CURRENT_VERSION
+ if ! [[ "${CURRENT_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+ printf " Please enter a valid x.y.z version\n"
+ return 1
+ fi
+ return 0
+}
+until getcurrentversion; do : ; done
+
+# version that we're building an RC for
+getversion() {
+ read -p " version being built: " VERSION
+ if ! [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+ printf " Please enter a valid x.y.z version\n"
+ return 1
+ fi
+ return 0
+ }
+until getversion; do : ; done
+
+# RC number we're building
+getrcnum() {
+read -p " release candidate number: " RC_NUM
+ if [[ "${RC_NUM}" =~ ^[0-9]+$ ]]; then
+ RC="${RC_PREFIX}${RC_NUM}"
+ return 0
+ else
+ printf " Please enter an integer\n"
+ return 1
+ fi
+}
+until getrcnum; do : ; done
+
+# define default values
+TMPDIR="$HOME/tmp"
+WORKDIR="$TMPDIR/${CHOSEN_REPO}-${VERSION}"
+
+# warn the user if the working directory exists
+if [ -d "$WORKDIR" ]; then
+ read -p " directory exists [$WORKDIR]. overwrite existing directory? [y/n] " -r
+ echo
+ if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+ exit 1
+ fi
+fi
+
+# Clear out the existing work directory
+rm -rf "$WORKDIR"
+mkdir "$WORKDIR"
+
+getbaserev() {
+ read -p " base revision branch or hash for release candidate [master]: " GIT_REF
+ GIT_REF=${GIT_REF:-master}
+
+ # check to see if we were given branch
+ git rev-parse -q --verify ${GIT_REF} >> /dev/null
+ if [[ $? -ne 0 ]]; then
+ # check to see if we were given hash
+ git cat-file -e ${GIT_REF}
+ if [[ $? -ne 0 ]]; then
+ "Unable to find git revision"
+ return 1
+ fi
+ fi
+ return 0
+}
+until getbaserev; do : ; done
+
+# Signing key
+getkey() {
+ read -s -p " signing key id in 8-byte format (e.g. BADDCAFEDEADBEEF): " SIGNING_KEY
+ printf "\n"
+ if ! [[ "${SIGNING_KEY}" =~ ^[A-F0-9]{16}$ ]]; then
+ printf " Please enter a valid signing key\n"
+ return 1
+ fi
+ return 0
+}
+
+until getkey; do : ; done
+
+# Determine if this is a practice run or not.
+getpractice() {
+ read -p " do a live run (push to remote repositories?) [y/n] " INPUT
+ case "${INPUT}" in
+ y)
+ PRACTICE_RUN=false
+ return 0 ;;
+ n)
+ PRACTICE_RUN=true
+ return 0 ;;
+ *)
+ printf "Please enter 'y' or 'n'\n"
+ return 1
+ ;;
+ esac
+}
+until getpractice; do : ; done
+
+## Prepare everything for building the release artifacts
+
+# Fetch the SVN repos. Always needed regardless of what's being released.
+fetch_svn_repo () {
+ local SVN_REPO=$1
+ local SVN_DIR=$2
+ mkdir "$SVN_DIR"
+ cd "$SVN_DIR"
+ printf "Checking out repo: %s\n" "$SVN_REPO" "$(basename $SVN_DIR)"
+ svn co -q $SVN_REPO
+ cd "$(dirname $WORKDIR)"
+}
+
+fetch_svn_repo "$DEV_REPO" "$WORKDIR/dev"
+fetch_svn_repo "$RELEASE_REPO" "$WORKDIR/release"
+
+# Fetch the appropriate Git repo. Only need what we're releasing
+GIT_DIR="$WORKDIR/${CHOSEN_REPO}"
+printf "Checking out git repo: %s\n" "$UPSTREAM"
+git clone $UPSTREAM "${GIT_DIR}"
+cd "${GIT_DIR}"
+git fetch --tags
+
+# Create the release branch in the Git repo
+printf "Creating branch: %s_%s\n" "${CAPITAL_REPO}" "$VERSION"
+printf "Using git rev: %s\n" ${GIT_REF}
+cd "$GIT_DIR"
+git checkout ${GIT_REF}
+git checkout -b "${CAPITAL_REPO}_${VERSION}"
+
+if [ "${PRACTICE_RUN}" = true ]; then
+ printf "This is a practice run. Not running <git push --set-upstream origin %s_%s>\n" "${CAPITAL_REPO}" "$VERSION"
+else
+ printf "Pushing branch %s_%s\n" "${CAPITAL_REPO}" "$VERSION"
+ git push --set-upstream origin "${BRANCH_PREFIX}_${BRANCH_VERSION}"
+fi
+
+# Create directory for release artifacts
+if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
+ ART_DIR="$WORKDIR/dev/metron/${VERSION}-RC${RC_NUM}"
+else
+ # We're using a sub module, so put it in it's own directory.
+ ART_DIR="$WORKDIR/dev/metron/${CHOSEN_REPO}/${VERSION}-RC${RC_NUM}"
+fi
+mkdir -p "$ART_DIR"
+
+# Setup various parameters we need for the release artifacts
+if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
+ CORE_PREFIX="apache-metron-"
+ ARTIFACT_PREFIX="${CORE_PREFIX}"
+ TAG_VERSION="${CURRENT_VERSION}"
+ TAG="${CORE_PREFIX}${TAG_VERSION}${TAG_POSTFIX}"
+elif [ "${CHOSEN_REPO}" = "${BRO_PLUGIN_REPO_NAME}" ]; then
+ PLUGIN_PREFIX="apache-metron-bro-plugin-kafka_"
+ ARTIFACT_PREFIX="${PLUGIN_PREFIX}"
+ TAG_VERSION="${CURRENT_VERSION}"
+ TAG="${PLUGIN_PREFIX}${TAG_VERSION}${TAG_POSTFIX}"
+ # Handle special tag case from prior release
+ if [ "${TAG_VERSION}" = "0.1" ]; then
+ TAG="0.1"
+ fi
+else
+ # If we ever add new modules, add them as needed.
+ printf "Unrecognized module: %s\n" "${CHOSEN_REPO}"
+ exit 1
+fi
+ARTIFACT="${ARTIFACT_PREFIX}${VERSION}-${RC}"
+
+## Do the work of actually creating the release artifacts
+printf "Creating tentative git tag <%s%s-%s>. Do not push this tag until RC is ready for community review.\n" "${TAG_PREFIX}" "$VERSION" "$RC"
+cd "$GIT_DIR"
+git checkout "${CAPITAL_REPO}_${VERSION}"
+# The branch only exists if this is not a practice run
+if [ "${PRACTICE_RUN}" = false ]; then
+ printf "Pulling latest state of branch\n"
+ git pull
+fi
+git tag "${ARTIFACT}"
+
+# Create the rc tarball from the tag
+printf "Creating the RC tarball for tag %s\n" "$ARTIFACT"
+git archive "--prefix=${ARTIFACT}/" "${ARTIFACT}" | gzip > "${ARTIFACT}.tar.gz"
+
+# Create signing hash files
+printf "Creating the SHA hash files\n"
+gpg --print-md SHA512 ${ARTIFACT}.tar.gz > ${ARTIFACT}.tar.gz.sha512
+gpg --print-md SHA256 ${ARTIFACT}.tar.gz > ${ARTIFACT}.tar.gz.sha256
+
+# Sign the release tarball
+printf "Signing the release tarball\n"
+gpg -u ${SIGNING_KEY} --armor --output ${ARTIFACT}.tar.gz.asc --detach-sig ${ARTIFACT}.tar.gz
+if [[ $? -ne 0 ]]; then
+ # gpg will print out an error on its own
+ exit 1
+fi
+
+# Setup the release artifacts
+printf "Copying release artifacts\n"
+mv "${GIT_DIR}/${ARTIFACT}.tar.gz" "$ART_DIR"
+mv "${ARTIFACT}.tar.gz.sha512" "$ART_DIR"
+mv "${ARTIFACT}.tar.gz.sha256" "$ART_DIR"
+mv "${ARTIFACT}.tar.gz.asc" "$ART_DIR"
+
+# Create the CHANGES file
+# Do this by getting all commits in current branch that aren't in current release. Filter out any merges by making sure lines start with blankspace followed by "METRON"
+# i.e. make sure the lines starts with a ticket number to avoid merge commits into feature branches
+printf "Creating CHANGES file\n"
+git log "${CAPITAL_REPO}_${VERSION}" "^tags/${TAG}" --no-merges | grep -E "^[[:blank:]]+METRON" | sed 's/\[//g' | sed 's/\]//g' | grep -v "http" > "${ART_DIR}/CHANGES"
+if [[ $? -ne 0 ]]; then
+ "Error creating CHANGES file"
+ exit 1
+fi
+
+printf "Extracting LICENSE, NOTICE, and KEYS from tarball\n" # Only pull from core
+cd ${ART_DIR}
+
+if [ "${CHOSEN_REPO}" = "${BRO_PLUGIN_REPO_NAME}" ]; then
+ # Bro's convention for the LICENSE is different, so the file is called COPYING
+ tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/COPYING"
+else
+ tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/LICENSE"
+fi
+
+# TODO figure out what to do for bro repo here. The KEYS file only needs to live in the /dist root, rather than in each sub repo.
+# Should we have a separate process for adding to the KEYS file without doing a Metron release?
+#tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/KEYS"
+tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/NOTICE"
+
+# Add the directory and commit to subversion
+COMMIT_DIR=$(basename ${ART_DIR})
+if [ "${PRACTICE_RUN}" = true ]; then
+ printf "This is a practice run. Not running the following commands:\n"
+ printf "<svn add %s>\n" ${COMMIT_DIR}
+ printf "<svn commit -m \"Adding artifacts for %s %s\">\n" "${CHOSEN_REPO}" "${COMMIT_DIR}"
+else
+ printf "Adding artifacts for Metron ${VERSION}-RC${RC_NUM} to dev SVN\n"
+ # Metron goes in the root of the dir, submodules go in folder
+ if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
+ cd "$WORKDIR/dev/metron/"
+ else
+ cd "$WORKDIR/dev/metron/${CHOSEN_REPO}"
+ fi
+ svn add ${COMMIT_DIR}
+ svn commit -m "Adding artifacts for ${CHOSEN_REPO} ${COMMIT_DIR}"
+fi
+
[49/51] [abbrv] metron git commit: METRON-1839 Install Elasticsearch
MPack Step in Ansible Not Idempotent (nickwallen) closes apache/metron#1244
Posted by ot...@apache.org.
METRON-1839 Install Elasticsearch MPack Step in Ansible Not Idempotent (nickwallen) closes apache/metron#1244
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/739782ab
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/739782ab
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/739782ab
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 739782ab71f6e571ff459d1844a77b533abbdbea
Parents: 0c0602c
Author: nickwallen <ni...@nickallen.org>
Authored: Mon Oct 22 15:39:25 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Mon Oct 22 15:39:25 2018 -0400
----------------------------------------------------------------------
metron-deployment/ansible/roles/ambari_master/defaults/main.yml | 1 +
.../ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml | 4 ++--
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/739782ab/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
index 8bae8ce..f6d76e5 100644
--- a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
+++ b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
@@ -19,5 +19,6 @@ ambari_server_mem: 2048
ambari_mpack_version: 0.6.1.0
metron_mpack_name: metron_mpack-{{ ambari_mpack_version }}.tar.gz
metron_mpack_path: "{{ playbook_dir }}/../../packaging/ambari/metron-mpack/target/{{ metron_mpack_name }}"
+elasticsearch_mpack_version: 5.6.2
elasticsearch_mpack_name: elasticsearch_mpack-{{ ambari_mpack_version }}.tar.gz
elasticsearch_mpack_path: "{{ playbook_dir }}/../../packaging/ambari/elasticsearch-mpack/target/{{ elasticsearch_mpack_name }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/739782ab/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml b/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
index 7ce1a13..ea5b0f4 100644
--- a/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
+++ b/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml
@@ -21,6 +21,6 @@
dest: /tmp
- name: Install Elasticsearch MPack on Ambari Host
- shell: ambari-server install-mpack --mpack=/tmp/elasticsearch_mpack-{{ ambari_mpack_version }}.tar.gz
+ shell: "ambari-server install-mpack --mpack=/tmp/{{ elasticsearch_mpack_name }}"
args:
- creates: /var/lib/ambari-server/resources/mpacks/elasticsearch-ambari.mpack-{{ ambari_mpack_version }}/addon-services
+ creates: /var/lib/ambari-server/resources/mpacks/elasticsearch-ambari.mpack-{{ elasticsearch_mpack_version }}/addon-services
[30/51] [abbrv] metron git commit: METRON-1792 Simplify Profile
Definitions in Integration Tests (nickwallen) closes apache/metron#1211
Posted by ot...@apache.org.
METRON-1792 Simplify Profile Definitions in Integration Tests (nickwallen) closes apache/metron#1211
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/5bfc08c5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/5bfc08c5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/5bfc08c5
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 5bfc08c57f1129b7d185ac7257197775ed3bdb5e
Parents: 269b91d
Author: nickwallen <ni...@nickallen.org>
Authored: Mon Oct 8 18:32:30 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Mon Oct 8 18:32:30 2018 -0400
----------------------------------------------------------------------
.../zookeeper/event-time-test/profiler.json | 19 ----
.../processing-time-test/profiler.json | 11 --
.../zookeeper/profile-with-stats/profiler.json | 12 --
.../integration/ConfigUploadComponent.java | 31 ++++--
.../integration/ProfilerIntegrationTest.java | 109 ++++++++++++++++---
.../ZKConfigurationsCacheIntegrationTest.java | 5 +-
.../src/test/resources/profiler/profiler.json | 19 ++++
7 files changed, 137 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/5bfc08c5/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json b/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json
deleted file mode 100644
index 534b7c6..0000000
--- a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "timestampField": "timestamp",
- "profiles": [
- {
- "profile": "count-by-ip",
- "foreach": "ip_src_addr",
- "init": { "count": 0 },
- "update": { "count" : "count + 1" },
- "result": "count"
- },
- {
- "profile": "total-count",
- "foreach": "'total'",
- "init": { "count": 0 },
- "update": { "count": "count + 1" },
- "result": "count"
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/5bfc08c5/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/processing-time-test/profiler.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/processing-time-test/profiler.json b/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/processing-time-test/profiler.json
deleted file mode 100644
index e75ec0f..0000000
--- a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/processing-time-test/profiler.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- "profiles": [
- {
- "profile": "processing-time-test",
- "foreach": "ip_src_addr",
- "init": { "counter": "0" },
- "update": { "counter": "counter + 1" },
- "result": "counter"
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/5bfc08c5/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/profile-with-stats/profiler.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/profile-with-stats/profiler.json b/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/profile-with-stats/profiler.json
deleted file mode 100644
index 083e73f..0000000
--- a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/profile-with-stats/profiler.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "profiles": [
- {
- "profile": "profile-with-stats",
- "foreach": "'global'",
- "init": { "stats": "STATS_INIT()" },
- "update": { "stats": "STATS_ADD(stats, 1)" },
- "result": "stats"
- }
- ],
- "timestampField": "timestamp"
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/5bfc08c5/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java
index 70487a0..eae3c52 100644
--- a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java
+++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ConfigUploadComponent.java
@@ -19,12 +19,15 @@
*/
package org.apache.metron.profiler.storm.integration;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
import org.apache.metron.integration.InMemoryComponent;
import org.apache.metron.integration.UnableToStartException;
import org.apache.metron.integration.components.ZKServerComponent;
+import java.util.Arrays;
import java.util.Properties;
import static org.apache.metron.common.configuration.ConfigurationsUtils.getClient;
@@ -41,7 +44,8 @@ public class ConfigUploadComponent implements InMemoryComponent {
private Properties topologyProperties;
private String globalConfiguration;
- private String profilerConfiguration;
+ private String profilerConfigurationPath;
+ private ProfilerConfig profilerConfig;
@Override
public void start() throws UnableToStartException {
@@ -86,11 +90,17 @@ public class ConfigUploadComponent implements InMemoryComponent {
* @param client The zookeeper client.
*/
private void uploadProfilerConfig(CuratorFramework client) throws Exception {
- if (profilerConfiguration != null) {
- byte[] globalConfig = readProfilerConfigFromFile(profilerConfiguration);
- if (globalConfig.length > 0) {
- writeProfilerConfigToZookeeper(readProfilerConfigFromFile(profilerConfiguration), client);
- }
+ byte[] configBytes = null;
+
+ if (profilerConfigurationPath != null) {
+ configBytes = readProfilerConfigFromFile(profilerConfigurationPath);
+
+ } else if(profilerConfig != null) {
+ configBytes = profilerConfig.toJSON().getBytes();
+ }
+
+ if (ArrayUtils.getLength(configBytes) > 0) {
+ writeProfilerConfigToZookeeper(configBytes, client);
}
}
@@ -117,8 +127,13 @@ public class ConfigUploadComponent implements InMemoryComponent {
return this;
}
- public ConfigUploadComponent withProfilerConfiguration(String path) {
- this.profilerConfiguration = path;
+ public ConfigUploadComponent withProfilerConfigurationPath(String path) {
+ this.profilerConfigurationPath = path;
+ return this;
+ }
+
+ public ConfigUploadComponent withProfilerConfiguration(ProfilerConfig profilerConfig) {
+ this.profilerConfig = profilerConfig;
return this;
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/5bfc08c5/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
index 64a1482..f7e75ce 100644
--- a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
+++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
@@ -23,6 +23,7 @@ package org.apache.metron.profiler.storm.integration;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.io.FileUtils;
import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
import org.apache.metron.hbase.mock.MockHBaseTableProvider;
import org.apache.metron.hbase.mock.MockHTable;
import org.apache.metron.integration.BaseIntegrationTest;
@@ -90,18 +91,15 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
public static final long startAt = 10;
public static final String entity = "10.0.0.1";
-
private static final String tableName = "profiler";
private static final String columnFamily = "P";
private static final String inputTopic = Constants.INDEXING_TOPIC;
private static final String outputTopic = "profiles";
private static final int saltDivisor = 10;
-
private static final long periodDurationMillis = TimeUnit.SECONDS.toMillis(20);
private static final long windowLagMillis = TimeUnit.SECONDS.toMillis(10);
private static final long windowDurationMillis = TimeUnit.SECONDS.toMillis(10);
private static final long profileTimeToLiveMillis = TimeUnit.SECONDS.toMillis(20);
-
private static final long maxRoutesPerBolt = 100000;
private static ZKServerComponent zkComponent;
@@ -110,11 +108,9 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
private static ConfigUploadComponent configUploadComponent;
private static ComponentRunner runner;
private static MockHTable profilerTable;
-
private static String message1;
private static String message2;
private static String message3;
-
private StellarStatefulExecutor executor;
/**
@@ -135,9 +131,25 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
@Multiline
private static String kryoSerializers;
+ /**
+ * {
+ * "profiles": [
+ * {
+ * "profile": "processing-time-test",
+ * "foreach": "ip_src_addr",
+ * "init": { "counter": "0" },
+ * "update": { "counter": "counter + 1" },
+ * "result": "counter"
+ * }
+ * ]
+ * }
+ */
+ @Multiline
+ private static String processingTimeProfile;
+
@Test
public void testProcessingTime() throws Exception {
- uploadConfigToZookeeper(TEST_RESOURCES + "/config/zookeeper/processing-time-test");
+ uploadConfigToZookeeper(ProfilerConfig.fromJSON(processingTimeProfile));
// start the topology and write 3 test messages to kafka
fluxComponent.submitTopology();
@@ -146,7 +158,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
kafkaComponent.writeMessages(inputTopic, message3);
// retrieve the profile measurement using PROFILE_GET
- String profileGetExpression = "PROFILE_GET('processing-time-test', '10.0.0.1', PROFILE_FIXED('5', 'MINUTES'))";
+ String profileGetExpression = "PROFILE_GET('processing-time-test', '10.0.0.1', PROFILE_FIXED('15', 'MINUTES'))";
List<Integer> measurements = execute(profileGetExpression, List.class);
// need to keep checking for measurements until the profiler has flushed one out
@@ -178,7 +190,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
@Test
public void testProcessingTimeWithTimeToLiveFlush() throws Exception {
- uploadConfigToZookeeper(TEST_RESOURCES + "/config/zookeeper/processing-time-test");
+ uploadConfigToZookeeper(ProfilerConfig.fromJSON(processingTimeProfile));
// start the topology and write 3 test messages to kafka
fluxComponent.submitTopology();
@@ -194,7 +206,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
kafkaComponent.writeMessages(inputTopic, message3);
// retrieve the profile measurement using PROFILE_GET
- String profileGetExpression = "PROFILE_GET('processing-time-test', '10.0.0.1', PROFILE_FIXED('5', 'MINUTES'))";
+ String profileGetExpression = "PROFILE_GET('processing-time-test', '10.0.0.1', PROFILE_FIXED('15', 'MINUTES'))";
List<Integer> measurements = execute(profileGetExpression, List.class);
// need to keep checking for measurements until the profiler has flushed one out
@@ -221,9 +233,33 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
assertEquals(3, measurements.get(0).intValue());
}
+ /**
+ * {
+ * "timestampField": "timestamp",
+ * "profiles": [
+ * {
+ * "profile": "count-by-ip",
+ * "foreach": "ip_src_addr",
+ * "init": { "count": 0 },
+ * "update": { "count" : "count + 1" },
+ * "result": "count"
+ * },
+ * {
+ * "profile": "total-count",
+ * "foreach": "'total'",
+ * "init": { "count": 0 },
+ * "update": { "count": "count + 1" },
+ * "result": "count"
+ * }
+ * ]
+ * }
+ */
+ @Multiline
+ private static String eventTimeProfile;
+
@Test
public void testEventTime() throws Exception {
- uploadConfigToZookeeper(TEST_RESOURCES + "/config/zookeeper/event-time-test");
+ uploadConfigToZookeeper(ProfilerConfig.fromJSON(eventTimeProfile));
// start the topology and write test messages to kafka
fluxComponent.submitTopology();
@@ -264,6 +300,23 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
}
/**
+ * {
+ * "profiles": [
+ * {
+ * "profile": "profile-with-stats",
+ * "foreach": "'global'",
+ * "init": { "stats": "STATS_INIT()" },
+ * "update": { "stats": "STATS_ADD(stats, 1)" },
+ * "result": "stats"
+ * }
+ * ],
+ * "timestampField": "timestamp"
+ * }
+ */
+ @Multiline
+ private static String profileWithStats;
+
+ /**
* The result produced by a Profile has to be serializable within Storm. If the result is not
* serializable the topology will crash and burn.
*
@@ -272,7 +325,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
*/
@Test
public void testProfileWithStatsObject() throws Exception {
- uploadConfigToZookeeper(TEST_RESOURCES + "/config/zookeeper/profile-with-stats");
+ uploadConfigToZookeeper(ProfilerConfig.fromJSON(profileWithStats));
// start the topology and write test messages to kafka
fluxComponent.submitTopology();
@@ -293,9 +346,34 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
assertTrue(results.get(0) instanceof OnlineStatisticsProvider);
}
+ /**
+ * {
+ * "profiles": [
+ * {
+ * "profile": "profile-with-triage",
+ * "foreach": "'global'",
+ * "update": {
+ * "stats": "STATS_ADD(stats, 1)"
+ * },
+ * "result": {
+ * "profile": "stats",
+ * "triage": {
+ * "min": "STATS_MIN(stats)",
+ * "max": "STATS_MAX(stats)",
+ * "mean": "STATS_MEAN(stats)"
+ * }
+ * }
+ * }
+ * ],
+ * "timestampField": "timestamp"
+ * }
+ */
+ @Multiline
+ private static String profileWithTriageResult;
+
@Test
public void testProfileWithTriageResult() throws Exception {
- uploadConfigToZookeeper(TEST_RESOURCES + "/config/zookeeper/triage-result");
+ uploadConfigToZookeeper(ProfilerConfig.fromJSON(profileWithTriageResult));
// start the topology and write test messages to kafka
fluxComponent.submitTopology();
@@ -466,13 +544,12 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
/**
* Uploads config values to Zookeeper.
- * @param path The path on the local filesystem to the config values.
+ * @param profilerConfig The Profiler configuration.
* @throws Exception
*/
- public void uploadConfigToZookeeper(String path) throws Exception {
+ public void uploadConfigToZookeeper(ProfilerConfig profilerConfig) throws Exception {
configUploadComponent
- .withGlobalConfiguration(path)
- .withProfilerConfiguration(path)
+ .withProfilerConfiguration(profilerConfig)
.update();
}
http://git-wip-us.apache.org/repos/asf/metron/blob/5bfc08c5/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java
index a7dc248..ce898d3 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java
@@ -119,7 +119,6 @@ public class ZKConfigurationsCacheIntegrationTest {
@Multiline
public static String globalConfig;
- public static File profilerDir = new File("../../metron-analytics/metron-profiler-storm/src/test/config/zookeeper");
public ConfigurationsCache cache;
public ZKServerComponent zkComponent;
@@ -154,7 +153,7 @@ public class ZKConfigurationsCacheIntegrationTest {
}
{
//profiler
- byte[] config = IOUtils.toByteArray(new FileInputStream(new File(profilerDir, "/event-time-test/profiler.json")));
+ byte[] config = IOUtils.toByteArray(new FileInputStream(new File("src/test/resources/profiler/profiler.json")));
ConfigurationsUtils.writeProfilerConfigToZookeeper( config, client);
}
{
@@ -284,7 +283,7 @@ public class ZKConfigurationsCacheIntegrationTest {
}
//profiler
{
- File inFile = new File(profilerDir, "/event-time-test/profiler.json");
+ File inFile = new File("src/test/resources/profiler/profiler.json");
ProfilerConfig expectedConfig = JSONUtils.INSTANCE.load(inFile, ProfilerConfig.class);
ProfilerConfigurations config = cache.get( ProfilerConfigurations.class);
assertEventually(() -> Assert.assertEquals(expectedConfig, config.getProfilerConfig()));
http://git-wip-us.apache.org/repos/asf/metron/blob/5bfc08c5/metron-platform/metron-common/src/test/resources/profiler/profiler.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/resources/profiler/profiler.json b/metron-platform/metron-common/src/test/resources/profiler/profiler.json
new file mode 100644
index 0000000..32be68d
--- /dev/null
+++ b/metron-platform/metron-common/src/test/resources/profiler/profiler.json
@@ -0,0 +1,19 @@
+{
+ "timestampField": "timestamp",
+ "profiles": [
+ {
+ "profile": "count-by-ip",
+ "foreach": "ip_src_addr",
+ "init": { "count": 0 },
+ "update": { "count" : "count + 1" },
+ "result": "count"
+ },
+ {
+ "profile": "total-count",
+ "foreach": "'total'",
+ "init": { "count": 0 },
+ "update": { "count": "count + 1" },
+ "result": "count"
+ }
+ ]
+}
[25/51] [abbrv] metron git commit: METRON-1771 Update REST endpoints
to support eventually consistent UI updates (merrimanr) closes
apache/metron#1190
Posted by ot...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
index 4d48075..096baf1 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
@@ -24,16 +24,17 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
-import java.util.function.Supplier;
import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
@@ -80,15 +81,16 @@ public abstract class AbstractLuceneMetaAlertUpdateDao implements MetaAlertUpdat
* @param retrieveLatestDao DAO to retrieve the item to be patched
* @param request The patch request.
* @param timestamp Optionally a timestamp to set. If not specified then current time is used.
+ * @return The patched document
* @throws OriginalNotFoundException If no original document is found to patch.
* @throws IOException If an error occurs performing the patch.
*/
@Override
- public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
+ public Document patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
Optional<Long> timestamp)
throws OriginalNotFoundException, IOException {
if (isPatchAllowed(request)) {
- updateDao.patch(retrieveLatestDao, request, timestamp);
+ return updateDao.patch(retrieveLatestDao, request, timestamp);
} else {
throw new IllegalArgumentException(
"Meta alert patches are not allowed for /alert or /status paths. "
@@ -97,7 +99,7 @@ public abstract class AbstractLuceneMetaAlertUpdateDao implements MetaAlertUpdat
}
@Override
- public void batchUpdate(Map<Document, Optional<String>> updates) {
+ public Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) {
throw new UnsupportedOperationException("Meta alerts do not allow for bulk updates");
}
@@ -170,21 +172,65 @@ public abstract class AbstractLuceneMetaAlertUpdateDao implements MetaAlertUpdat
return updates;
}
+ /**
+ * Adds alerts to a metaalert, based on a list of GetRequests provided for retrieval.
+ * @param metaAlertGuid The GUID of the metaalert to be given new children.
+ * @param alertRequests GetRequests for the appropriate alerts to add.
+ * @return The updated metaalert with alerts added.
+ */
+ @Override
+ public Document addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
+ throws IOException {
+ Document metaAlert = retrieveLatestDao
+ .getLatest(metaAlertGuid, MetaAlertConstants.METAALERT_TYPE);
+ if (metaAlert == null) {
+ throw new IOException(String.format("Unable to add alerts to meta alert. Meta alert with guid %s cannot be found.",
+ metaAlertGuid));
+ }
+ if (MetaAlertStatus.ACTIVE.getStatusString()
+ .equals(metaAlert.getDocument().get(MetaAlertConstants.STATUS_FIELD))) {
+ Iterable<Document> alerts = retrieveLatestDao.getAllLatest(alertRequests);
+ Set<String> missingAlerts = getMissingAlerts(alertRequests, alerts);
+ if (!missingAlerts.isEmpty()) {
+ throw new IOException(String.format("Unable to add alerts to meta alert. Alert with guid %s cannot be found.",
+ missingAlerts.iterator().next()));
+ }
+ Map<Document, Optional<String>> updates = buildAddAlertToMetaAlertUpdates(metaAlert, alerts);
+ update(updates);
+ return metaAlert;
+ } else {
+ throw new IllegalStateException("Adding alerts to an INACTIVE meta alert is not allowed");
+ }
+ }
+
+ /**
+ * Removes alerts from a metaalert, based on a list of GetRequests provided for retrieval.
+ * @param metaAlertGuid The GUID of the metaalert to remove children from.
+ * @param alertRequests A list of GetReqests that will provide the alerts to remove
+ * @return The updated metaalert with alerts removed.
+ * @throws IllegalStateException If the metaalert is inactive.
+ */
@Override
@SuppressWarnings("unchecked")
- public boolean removeAlertsFromMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
- throws IOException {
+ public Document removeAlertsFromMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
+ throws IOException, IllegalStateException {
Document metaAlert = retrieveLatestDao
.getLatest(metaAlertGuid, MetaAlertConstants.METAALERT_TYPE);
if (metaAlert == null) {
- return false;
+ throw new IOException(String.format("Unable to remove alerts from meta alert. Meta alert with guid %s cannot be found.",
+ metaAlertGuid));
}
if (MetaAlertStatus.ACTIVE.getStatusString()
.equals(metaAlert.getDocument().get(MetaAlertConstants.STATUS_FIELD))) {
Iterable<Document> alerts = retrieveLatestDao.getAllLatest(alertRequests);
+ Set<String> missingAlerts = getMissingAlerts(alertRequests, alerts);
+ if (!missingAlerts.isEmpty()) {
+ throw new IOException(String.format("Unable to remove alerts from meta alert. Alert with guid %s cannot be found.",
+ missingAlerts.iterator().next()));
+ }
Map<Document, Optional<String>> updates = buildRemoveAlertsFromMetaAlert(metaAlert, alerts);
update(updates);
- return updates.size() != 0;
+ return metaAlert;
} else {
throw new IllegalStateException("Removing alerts from an INACTIVE meta alert is not allowed");
}
@@ -213,10 +259,14 @@ public abstract class AbstractLuceneMetaAlertUpdateDao implements MetaAlertUpdat
}
@Override
- public boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status)
+ public Document updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status)
throws IOException {
Document metaAlert = retrieveLatestDao
.getLatest(metaAlertGuid, MetaAlertConstants.METAALERT_TYPE);
+ if (metaAlert == null) {
+ throw new IOException(String.format("Unable to update meta alert status. Meta alert with guid %s cannot be found.",
+ metaAlertGuid));
+ }
String currentStatus = (String) metaAlert.getDocument().get(MetaAlertConstants.STATUS_FIELD);
boolean metaAlertUpdated = !status.getStatusString().equals(currentStatus);
if (metaAlertUpdated) {
@@ -231,7 +281,7 @@ public abstract class AbstractLuceneMetaAlertUpdateDao implements MetaAlertUpdat
Map<Document, Optional<String>> updates = buildStatusChangeUpdates(metaAlert, alerts, status);
update(updates);
}
- return metaAlertUpdated;
+ return metaAlert;
}
/**
@@ -334,4 +384,13 @@ public abstract class AbstractLuceneMetaAlertUpdateDao implements MetaAlertUpdat
} // else we have no updates, so don't do anything
}
+ protected Set<String> getMissingAlerts(List<GetRequest> alertRequests, Iterable<Document> results) throws IOException {
+ Set<String> requestGuids = alertRequests.stream().map(GetRequest::getGuid).collect(Collectors.toSet());
+ Set<String> resultGuids = StreamSupport.stream(results.spliterator(), false)
+ .map(Document::getGuid).collect(Collectors.toSet());
+ Set<String> missingGuids = new HashSet<>(requestGuids);
+ missingGuids.removeAll(resultGuids);
+ return missingGuids;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
index b5f38e4..8f6f6b0 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
@@ -18,7 +18,6 @@
package org.apache.metron.indexing.dao.update;
import java.io.IOException;
-import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import org.apache.metron.common.utils.JSONUtils;
@@ -33,25 +32,27 @@ public interface UpdateDao {
*
* @param update The document to replace from the index.
* @param index The index where the document lives.
+ * @return The updated document
* @throws IOException If an error occurs during the update.
*/
- void update(Document update, Optional<String> index) throws IOException;
+ Document update(Document update, Optional<String> index) throws IOException;
/**
* Similar to the update method but accepts multiple documents and performs updates in batch.
*
* @param updates A map of the documents to update to the index where they live.
+ * @return The updated documents.
* @throws IOException If an error occurs during the updates.
*/
- void batchUpdate(Map<Document, Optional<String>> updates) throws IOException;
+ Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) throws IOException;
- void addCommentToAlert(CommentAddRemoveRequest request) throws IOException;
+ Document addCommentToAlert(CommentAddRemoveRequest request) throws IOException;
- void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException;
+ Document removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException;
- void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException;
+ Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException;
- void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException;
+ Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException;
/**
@@ -59,14 +60,15 @@ public interface UpdateDao {
* https://tools.ietf.org/html/rfc6902)
* @param request The patch request
* @param timestamp Optionally a timestamp to set. If not specified then current time is used.
+ * @return The patched document.
* @throws OriginalNotFoundException If the original is not found, then it cannot be patched.
* @throws IOException If an error occurs while patching.
*/
- default void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request
+ default Document patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request
, Optional<Long> timestamp
) throws OriginalNotFoundException, IOException {
Document d = getPatchedDocument(retrieveLatestDao, request, timestamp);
- update(d, Optional.ofNullable(request.getIndex()));
+ return update(d, Optional.ofNullable(request.getIndex()));
}
default Document getPatchedDocument(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
@@ -94,15 +96,16 @@ public interface UpdateDao {
* Replace a document in an index.
* @param request The replacement request.
* @param timestamp The timestamp (optional) of the update. If not specified, then current time will be used.
+ * @return The replaced document.
* @throws IOException If an error occurs during replacement.
*/
- default void replace(ReplaceRequest request, Optional<Long> timestamp)
+ default Document replace(ReplaceRequest request, Optional<Long> timestamp)
throws IOException {
Document d = new Document(request.getReplacement(),
request.getGuid(),
request.getSensorType(),
timestamp.orElse(System.currentTimeMillis())
);
- update(d, Optional.ofNullable(request.getIndex()));
+ return update(d, Optional.ofNullable(request.getIndex()));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/InMemoryMetaAlertRetrieveLatestDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/InMemoryMetaAlertRetrieveLatestDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/InMemoryMetaAlertRetrieveLatestDao.java
new file mode 100644
index 0000000..28891d9
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/InMemoryMetaAlertRetrieveLatestDao.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.indexing;
+
+import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertRetrieveLatestDao;
+import org.apache.metron.indexing.dao.search.GetRequest;
+import org.apache.metron.indexing.dao.update.Document;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * This class is needed to compose an InMemoryMetaAlertUpdateDao implementation. This allows the
+ * InMemoryMetaAlertUpdateDao class to extend AbstractLuceneMetaAlertUpdateDao and reuse common logic in that class.
+ */
+public class InMemoryMetaAlertRetrieveLatestDao implements MetaAlertRetrieveLatestDao {
+
+ private IndexDao indexDao;
+
+ public InMemoryMetaAlertRetrieveLatestDao(IndexDao indexDao) {
+ this.indexDao = indexDao;
+ }
+
+ @Override
+ public Document getLatest(String guid, String sensorType) throws IOException {
+ return indexDao.getLatest(guid, sensorType);
+ }
+
+ @Override
+ public Iterable<Document> getAllLatest(List<GetRequest> getRequests) throws IOException {
+ return indexDao.getAllLatest(getRequests);
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/HBaseDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/HBaseDaoTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/HBaseDaoTest.java
new file mode 100644
index 0000000..d4823db
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/HBaseDaoTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao;
+
+import org.apache.metron.indexing.dao.update.UpdateDao;
+import org.junit.Before;
+
+/**
+ * This class returns the HBaseDao implementation to be used in UpdateDaoTest. UpdateDaoTest contains a
+ * common set of tests that all Dao implementations must pass.
+ */
+public class HBaseDaoTest extends UpdateDaoTest{
+
+ private HBaseDao dao;
+
+ @Before
+ public void setup() {
+ dao = new HBaseDao();
+ }
+
+ @Override
+ public UpdateDao getUpdateDao() {
+ return dao;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
index e306567..f49a6ad 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
@@ -248,7 +248,7 @@ public class InMemoryDao implements IndexDao {
}
@Override
- public void update(Document update, Optional<String> index) throws IOException {
+ public Document update(Document update, Optional<String> index) throws IOException {
for (Map.Entry<String, List<String>> kv : BACKING_STORE.entrySet()) {
if (kv.getKey().startsWith(update.getSensorType())) {
for (Iterator<String> it = kv.getValue().iterator(); it.hasNext(); ) {
@@ -261,13 +261,15 @@ public class InMemoryDao implements IndexDao {
kv.getValue().add(JSONUtils.INSTANCE.toJSON(update.getDocument(), true));
}
}
+ return update;
}
@Override
- public void batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
+ public Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
for (Map.Entry<Document, Optional<String>> update : updates.entrySet()) {
update(update.getKey(), update.getValue());
}
+ return updates;
}
@Override
@@ -293,19 +295,23 @@ public class InMemoryDao implements IndexDao {
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) {
+ public Document addCommentToAlert(CommentAddRemoveRequest request) {
+ return null;
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request) {
+ return null;
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ return null;
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ return null;
}
public static void setColumnMetadata(Map<String, Map<String, FieldType>> columnMetadata) {
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
index cb8837b..dbd3cb6 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
@@ -18,26 +18,24 @@
package org.apache.metron.indexing.dao;
-import static org.apache.metron.common.Constants.GUID;
+import static org.apache.metron.common.Constants.SENSOR_TYPE;
-import com.google.common.collect.ImmutableList;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.stream.Collectors;
+import java.util.function.Supplier;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.indexing.InMemoryMetaAlertRetrieveLatestDao;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
-import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
import org.apache.metron.indexing.dao.metaalert.MetaAlertDao;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertRetrieveLatestDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
-import org.apache.metron.indexing.dao.metaalert.MetaScores;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertUpdateDao;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.GroupRequest;
@@ -46,20 +44,19 @@ import org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
-import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
public class InMemoryMetaAlertDao implements MetaAlertDao {
- public static Map<String, Collection<String>> METAALERT_STORE = new HashMap<>();
+ public static final String METAALERT_INDEX = "metaalert_index";
private IndexDao indexDao;
+ private MetaAlertRetrieveLatestDao metaAlertRetrieveLatestDao;
+ private MetaAlertUpdateDao metaAlertUpdateDao;
private int pageSize = 10;
/**
@@ -97,6 +94,24 @@ public class InMemoryMetaAlertDao implements MetaAlertDao {
@Override
public void init(IndexDao indexDao, Optional<String> threatSort) {
this.indexDao = indexDao;
+ this.metaAlertRetrieveLatestDao = new InMemoryMetaAlertRetrieveLatestDao(indexDao);
+ Supplier<Map<String, Object>> globalConfigSupplier = () -> new HashMap<>();
+ MetaAlertConfig config = new MetaAlertConfig(
+ METAALERT_INDEX,
+ null,
+ globalConfigSupplier
+ ) {
+ @Override
+ protected String getDefaultThreatTriageField() {
+ return MetaAlertConstants.THREAT_FIELD_DEFAULT;
+ }
+
+ @Override
+ protected String getDefaultSourceTypeField() {
+ return SENSOR_TYPE;
+ }
+ };
+ this.metaAlertUpdateDao = new InMemoryMetaAlertUpdateDao(indexDao, metaAlertRetrieveLatestDao, config, -1);
// Ignore threatSort for test.
}
@@ -112,12 +127,12 @@ public class InMemoryMetaAlertDao implements MetaAlertDao {
}
@Override
- public void update(Document update, Optional<String> index) throws IOException {
- indexDao.update(update, index);
+ public Document update(Document update, Optional<String> index) throws IOException {
+ return indexDao.update(update, index);
}
@Override
- public void batchUpdate(Map<Document, Optional<String>> updates) {
+ public Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) {
throw new UnsupportedOperationException("InMemoryMetaAlertDao can't do bulk updates");
}
@@ -128,19 +143,23 @@ public class InMemoryMetaAlertDao implements MetaAlertDao {
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) {
+ public Document addCommentToAlert(CommentAddRemoveRequest request) {
+ return null;
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request) {
+ return null;
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ return null;
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ return null;
}
@Override
@@ -149,15 +168,15 @@ public class InMemoryMetaAlertDao implements MetaAlertDao {
}
@Override
- public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
+ public Document patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
Optional<Long> timestamp)
throws OriginalNotFoundException, IOException {
- indexDao.patch(retrieveLatestDao, request, timestamp);
+ return indexDao.patch(retrieveLatestDao, request, timestamp);
}
@Override
- public void replace(ReplaceRequest request, Optional<Long> timestamp) throws IOException {
- indexDao.replace(request, timestamp);
+ public Document replace(ReplaceRequest request, Optional<Long> timestamp) throws IOException {
+ return indexDao.replace(request, timestamp);
}
@Override
@@ -174,145 +193,29 @@ public class InMemoryMetaAlertDao implements MetaAlertDao {
@SuppressWarnings("unchecked")
@Override
- public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request)
- throws InvalidCreateException {
- List<GetRequest> alertRequests = request.getAlerts();
- if (alertRequests.isEmpty()) {
- MetaAlertCreateResponse response = new MetaAlertCreateResponse();
- response.setCreated(false);
- return response;
- }
- // Build meta alert json. Give it a reasonable GUID
- JSONObject metaAlert = new JSONObject();
- String metaAlertGuid =
- "meta_" + (InMemoryDao.BACKING_STORE.get(getMetaAlertIndex()).size() + 1);
- metaAlert.put(GUID, metaAlertGuid);
-
- JSONArray groupsArray = new JSONArray();
- groupsArray.addAll(request.getGroups());
- metaAlert.put(MetaAlertConstants.GROUPS_FIELD, groupsArray);
-
- // Retrieve the alert for each guid
- // For the purpose of testing, we're just using guids for the alerts field and grabbing the scores.
- JSONArray alertArray = new JSONArray();
- List<Double> threatScores = new ArrayList<>();
- Collection<String> alertGuids = new ArrayList<>();
- for (GetRequest alertRequest : alertRequests) {
- SearchRequest searchRequest = new SearchRequest();
- searchRequest.setIndices(ImmutableList.of(alertRequest.getIndex().get()));
- searchRequest.setQuery("guid:" + alertRequest.getGuid());
- try {
- SearchResponse searchResponse = search(searchRequest);
- List<SearchResult> searchResults = searchResponse.getResults();
- if (searchResults.size() > 1) {
- throw new InvalidCreateException(
- "Found more than one result for: " + alertRequest.getGuid() + ". Values: "
- + searchResults
- );
- }
-
- if (searchResults.size() == 1) {
- SearchResult result = searchResults.get(0);
- alertArray.add(result.getSource());
- Double threatScore = Double
- .parseDouble(
- result.getSource().getOrDefault(MetaAlertConstants.THREAT_FIELD_DEFAULT, "0")
- .toString());
-
- threatScores.add(threatScore);
- }
- } catch (InvalidSearchException e) {
- throw new InvalidCreateException("Unable to find guid: " + alertRequest.getGuid(), e);
- }
- alertGuids.add(alertRequest.getGuid());
- }
-
- metaAlert.put(MetaAlertConstants.ALERT_FIELD, alertArray);
- metaAlert.putAll(new MetaScores(threatScores).getMetaScores());
- metaAlert.put(MetaAlertConstants.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString());
-
- // Add the alert to the store, but make sure not to overwrite existing results
- InMemoryDao.BACKING_STORE.get(getMetaAlertIndex()).add(metaAlert.toJSONString());
-
- METAALERT_STORE.put(metaAlertGuid, new HashSet<>(alertGuids));
-
- MetaAlertCreateResponse createResponse = new MetaAlertCreateResponse();
- createResponse.setGuid(metaAlertGuid);
- createResponse.setCreated(true);
- return createResponse;
+ public Document createMetaAlert(MetaAlertCreateRequest request)
+ throws InvalidCreateException, IOException {
+ return metaAlertUpdateDao.createMetaAlert(request);
}
@Override
- public boolean addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests) {
- Collection<String> currentAlertGuids = METAALERT_STORE.get(metaAlertGuid);
- if (currentAlertGuids == null) {
- return false;
- }
- Collection<String> alertGuids = alertRequests.stream().map(GetRequest::getGuid)
- .collect(Collectors.toSet());
- boolean added = currentAlertGuids.addAll(alertGuids);
- if (added) {
- METAALERT_STORE.put(metaAlertGuid, currentAlertGuids);
- }
- return added;
+ public Document addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests) throws IOException {
+ return metaAlertUpdateDao.addAlertsToMetaAlert(metaAlertGuid, alertRequests);
}
@Override
- public boolean removeAlertsFromMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests) {
- Collection<String> currentAlertGuids = METAALERT_STORE.get(metaAlertGuid);
- if (currentAlertGuids == null) {
- return false;
- }
- Collection<String> alertGuids = alertRequests.stream().map(GetRequest::getGuid)
- .collect(Collectors.toSet());
- boolean removed = currentAlertGuids.removeAll(alertGuids);
- if (removed) {
- METAALERT_STORE.put(metaAlertGuid, currentAlertGuids);
- }
- return removed;
+ public Document removeAlertsFromMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests) throws IOException {
+ return metaAlertUpdateDao.removeAlertsFromMetaAlert(metaAlertGuid, alertRequests);
}
@SuppressWarnings("unchecked")
@Override
- public boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status)
+ public Document updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status)
throws IOException {
- boolean statusChanged = false;
- List<String> metaAlerts = InMemoryDao.BACKING_STORE.get(getMetaAlertIndex());
- for (String metaAlert : metaAlerts) {
- JSONObject metaAlertJSON = JSONUtils.INSTANCE.load(metaAlert, JSONObject.class);
- if (metaAlertGuid.equals(metaAlertJSON.get(GUID))) {
- statusChanged = !status.getStatusString()
- .equals(metaAlertJSON.get(MetaAlertConstants.STATUS_FIELD));
- if (statusChanged) {
- metaAlertJSON.put(MetaAlertConstants.STATUS_FIELD, status.getStatusString());
- metaAlerts.remove(metaAlert);
- metaAlerts.add(metaAlertJSON.toJSONString());
- InMemoryDao.BACKING_STORE.put(getMetaAlertIndex(), metaAlerts);
- }
- break;
- }
- }
- return statusChanged;
- }
-
- public int getPageSize() {
- return pageSize;
- }
-
- public void setPageSize(int pageSize) {
- this.pageSize = pageSize;
- }
-
- public String getMetAlertSensorName() {
- return MetaAlertConstants.METAALERT_TYPE;
- }
-
- public String getMetaAlertIndex() {
- return "metaalert_index";
+ return metaAlertUpdateDao.updateMetaAlertStatus(metaAlertGuid, status);
}
public static void clear() {
InMemoryDao.clear();
- METAALERT_STORE.clear();
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertUpdateDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertUpdateDao.java
new file mode 100644
index 0000000..53564b0
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertUpdateDao.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.indexing.dao;
+
+import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertRetrieveLatestDao;
+import org.apache.metron.indexing.dao.metaalert.lucene.AbstractLuceneMetaAlertUpdateDao;
+import org.apache.metron.indexing.dao.search.GetRequest;
+import org.apache.metron.indexing.dao.search.InvalidCreateException;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
+import org.apache.metron.indexing.dao.update.Document;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Optional;
+
+public class InMemoryMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao {
+
+ private IndexDao indexDao;
+
+ public InMemoryMetaAlertUpdateDao(
+ IndexDao indexDao,
+ MetaAlertRetrieveLatestDao retrieveLatestDao,
+ MetaAlertConfig config,
+ int pageSize
+ ) {
+ super(indexDao, retrieveLatestDao, config);
+ this.indexDao = indexDao;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Document createMetaAlert(MetaAlertCreateRequest request) throws InvalidCreateException, IOException {
+ List<GetRequest> alertRequests = request.getAlerts();
+ if (alertRequests.isEmpty()) {
+ return null;
+ }
+ // Retrieve the documents going into the meta alert and build it
+ Iterable<Document> alerts = indexDao.getAllLatest(alertRequests);
+
+ Document metaAlert = buildCreateDocument(alerts, request.getGroups(),
+ MetaAlertConstants.ALERT_FIELD);
+
+ metaAlert.getDocument()
+ .put(getConfig().getSourceTypeField(), MetaAlertConstants.METAALERT_TYPE);
+
+ return metaAlert;
+ }
+
+ @Override
+ public Document update(Document update, Optional<String> index) throws IOException {
+ return indexDao.update(update, index);
+ }
+
+ @Override
+ public Document addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ return null;
+ }
+
+ @Override
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ return null;
+ }
+
+ @Override
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ return null;
+ }
+
+ @Override
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/MultiIndexDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/MultiIndexDaoTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/MultiIndexDaoTest.java
new file mode 100644
index 0000000..dad6a52
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/MultiIndexDaoTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao;
+
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
+import org.apache.metron.indexing.dao.update.Document;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class MultiIndexDaoTest {
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ private MultiIndexDao multiIndexDao;
+ private IndexDao dao1;
+ private IndexDao dao2;
+
+ @Before
+ public void setup() {
+ dao1 = mock(IndexDao.class);
+ dao2 = mock(IndexDao.class);
+ multiIndexDao = new MultiIndexDao(dao1, dao2);
+ }
+
+ @Test
+ public void getLatestShouldReturnLatestAlert() throws Exception {
+ Document document1 = new Document(new HashMap<>(), "guid", "bro", 1L);
+ Document document2 = new Document(new HashMap<>(), "guid", "bro", 2L);
+
+ CommentAddRemoveRequest request = new CommentAddRemoveRequest();
+ request.setGuid("guid");
+ when(dao1.getLatest("guid", "bro")).thenReturn(document1);
+ when(dao2.getLatest("guid", "bro")).thenReturn(document2);
+
+
+ Document expected = new Document(new HashMap<>(), "guid", "bro", 2L);
+ Assert.assertEquals(expected, multiIndexDao.getLatest("guid", "bro"));
+ }
+
+ @Test
+ public void addCommentShouldAddCommentToAlert() throws Exception {
+ Document latest = mock(Document.class);
+ Document document1 = new Document(new HashMap<>(), "guid", "bro", 1L);
+ Document document2 = new Document(new HashMap<>(), "guid", "bro", 2L);
+
+ CommentAddRemoveRequest request = new CommentAddRemoveRequest();
+ request.setGuid("guid");
+ when(dao1.addCommentToAlert(request, latest)).thenReturn(document1);
+ when(dao2.addCommentToAlert(request, latest)).thenReturn(document2);
+
+
+ Document expected = new Document(new HashMap<>(), "guid", "bro", 2L);
+ Assert.assertEquals(expected, multiIndexDao.addCommentToAlert(request, latest));
+ }
+
+ @Test
+ public void removeCommentShouldRemoveCommentFromAlert() throws Exception {
+ Document latest = mock(Document.class);
+ Document document1 = new Document(new HashMap<>(), "guid", "bro", 1L);
+ Document document2 = new Document(new HashMap<>(), "guid", "bro", 2L);
+
+ CommentAddRemoveRequest request = new CommentAddRemoveRequest();
+ request.setGuid("guid");
+ when(dao1.removeCommentFromAlert(request, latest)).thenReturn(document1);
+ when(dao2.removeCommentFromAlert(request, latest)).thenReturn(document2);
+
+
+ Document expected = new Document(new HashMap<>(), "guid", "bro", 2L);
+ Assert.assertEquals(expected, multiIndexDao.removeCommentFromAlert(request, latest));
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateDaoTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateDaoTest.java
new file mode 100644
index 0000000..bbe7fa8
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateDaoTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao;
+
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
+import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.update.UpdateDao;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+/**
+ * The tests in this class are common among all UpdateDao implementations.
+ */
+public abstract class UpdateDaoTest {
+
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ @Test
+ public void addCommentShouldThrowExceptionOnMissingAlert() throws Exception {
+ exception.expect(IOException.class);
+ exception.expectMessage("Unable to add comment. Document with guid guid cannot be found.");
+
+ CommentAddRemoveRequest request = new CommentAddRemoveRequest();
+ request.setGuid("guid");
+
+ getUpdateDao().addCommentToAlert(request, null);
+ }
+
+ @Test
+ public void removeCommentShouldThrowExceptionOnMissingAlert() throws Exception {
+ exception.expect(IOException.class);
+ exception.expectMessage("Unable to remove comment. Document with guid guid cannot be found.");
+
+ CommentAddRemoveRequest request = new CommentAddRemoveRequest();
+ request.setGuid("guid");
+
+ getUpdateDao().removeCommentFromAlert(request, null);
+ }
+
+ @Test
+ public void removeCommentShouldThrowExceptionOnEmptyComments() throws Exception {
+ exception.expect(IOException.class);
+ exception.expectMessage("Unable to remove comment. Document with guid guid has no comments.");
+
+ CommentAddRemoveRequest request = new CommentAddRemoveRequest();
+ request.setGuid("guid");
+ Document latest = new Document(new HashMap<>(), "guid", "bro", System.currentTimeMillis());
+
+ getUpdateDao().removeCommentFromAlert(request, latest);
+ }
+
+ public abstract UpdateDao getUpdateDao();
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
index 1e35523..ef9714e 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
@@ -18,7 +18,6 @@ import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -99,13 +98,14 @@ public abstract class UpdateIntegrationTest {
put("new-field", "metron");
}};
String guid = "" + message0.get(Constants.GUID);
- getDao().replace(new ReplaceRequest(){{
+ Document update = getDao().replace(new ReplaceRequest(){{
setReplacement(message0);
setGuid(guid);
setSensorType(SENSOR_NAME);
setIndex(getIndexName());
}}, Optional.empty());
+ Assert.assertEquals(message0, update.getDocument());
Assert.assertEquals(1, getMockHTable().size());
findUpdatedDoc(message0, guid, SENSOR_NAME);
{
@@ -138,12 +138,13 @@ public abstract class UpdateIntegrationTest {
put("new-field", "metron2");
}};
String guid = "" + message0.get(Constants.GUID);
- getDao().replace(new ReplaceRequest(){{
+ Document update = getDao().replace(new ReplaceRequest(){{
setReplacement(message0);
setGuid(guid);
setSensorType(SENSOR_NAME);
setIndex(getIndexName());
}}, Optional.empty());
+ Assert.assertEquals(message0, update.getDocument());
Assert.assertEquals(1, getMockHTable().size());
Document doc = getDao().getLatest(guid, SENSOR_NAME);
Assert.assertEquals(message0, doc.getDocument());
@@ -184,33 +185,40 @@ public abstract class UpdateIntegrationTest {
fields.put("source.type", SENSOR_NAME);
Document document = new Document(fields, "add_comment", SENSOR_NAME, 1526306463050L);
- getDao().update(document, Optional.of(SENSOR_NAME));
- findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
-
- addAlertComment("add_comment", "New Comment", "test_user", 1526306463050L);
- // Ensure we have the first comment
+ {
+ Document update = getDao().update(document, Optional.of(SENSOR_NAME));
+ Assert.assertEquals(document, update);
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ }
ArrayList<AlertComment> comments = new ArrayList<>();
- comments.add(new AlertComment("New Comment", "test_user", 1526306463050L));
- document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
- Collectors.toList()));
- findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
-
- List<Map<String, Object>> patchList = new ArrayList<>();
- Map<String, Object> patch = new HashMap<>();
- patch.put("op", "add");
- patch.put("path", "/project");
- patch.put("value", "metron");
- patchList.add(patch);
+ {
+ Document update = addAlertComment("add_comment", "New Comment", "test_user", 1526306463050L);
+ // Ensure we have the first comment
+ comments.add(new AlertComment("New Comment", "test_user", 1526306463050L));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ Assert.assertEquals(document, update);
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ }
+ {
+ List<Map<String, Object>> patchList = new ArrayList<>();
+ Map<String, Object> patch = new HashMap<>();
+ patch.put("op", "add");
+ patch.put("path", "/project");
+ patch.put("value", "metron");
+ patchList.add(patch);
- PatchRequest pr = new PatchRequest();
- pr.setGuid("add_comment");
- pr.setIndex(SENSOR_NAME);
- pr.setSensorType(SENSOR_NAME);
- pr.setPatch(patchList);
- getDao().patch(getDao(), pr, Optional.of(new Date().getTime()));
+ PatchRequest pr = new PatchRequest();
+ pr.setGuid("add_comment");
+ pr.setIndex(SENSOR_NAME);
+ pr.setSensorType(SENSOR_NAME);
+ pr.setPatch(patchList);
+ Document update = getDao().patch(getDao(), pr, Optional.of(1526306463050L));
- document.getDocument().put("project", "metron");
- findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ document.getDocument().put("project", "metron");
+ Assert.assertEquals(document, update);
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ }
}
@Test
@@ -221,48 +229,60 @@ public abstract class UpdateIntegrationTest {
fields.put("source.type", SENSOR_NAME);
Document document = new Document(fields, "add_comment", SENSOR_NAME, 1526401584951L);
- getDao().update(document, Optional.of(SENSOR_NAME));
- findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
-
- addAlertComment("add_comment", "New Comment", "test_user", 1526401584951L);
- // Ensure we have the first comment
+ {
+ Document update = getDao().update(document, Optional.of(SENSOR_NAME));
+ Assert.assertEquals(document, update);
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ }
ArrayList<AlertComment> comments = new ArrayList<>();
- comments.add(new AlertComment("New Comment", "test_user", 1526401584951L));
- document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
- Collectors.toList()));
- findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
-
- addAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L);
- // Ensure we have the second comment
- comments.add(new AlertComment("New Comment 2", "test_user_2", 1526401584952L));
- document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
- Collectors.toList()));
- findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
-
- removeAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L);
- // Ensure we only have the first comments
- comments = new ArrayList<>();
- comments.add(new AlertComment(commentOne));
- document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
- Collectors.toList()));
- findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ {
+ Document update = addAlertComment("add_comment", "New Comment", "test_user", 1526401584951L);
+ // Ensure we have the first comment
- removeAlertComment("add_comment", "New Comment", "test_user", 1526401584951L);
- // Ensure we have no comments
- document.getDocument().remove(COMMENTS_FIELD);
- findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ comments.add(new AlertComment("New Comment", "test_user", 1526401584951L));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ Assert.assertEquals(document, update);
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ }
+ {
+ Document update = addAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L);
+ // Ensure we have the second comment
+ comments.add(new AlertComment("New Comment 2", "test_user_2", 1526401584952L));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ Assert.assertEquals(document, update);
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ }
+ {
+ Document update = removeAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L);
+ // Ensure we only have the first comments
+ comments = new ArrayList<>();
+ comments.add(new AlertComment(commentOne));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ Assert.assertEquals(document, update);
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ }
+ {
+ Document update = removeAlertComment("add_comment", "New Comment", "test_user", 1526401584951L);
+ // Ensure we have no comments
+ document.getDocument().remove(COMMENTS_FIELD);
+ Assert.assertEquals(document, update);
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ }
}
- protected void addAlertComment(String guid, String comment, String username, long timestamp)
+ protected Document addAlertComment(String guid, String comment, String username, long timestamp)
throws IOException {
CommentAddRemoveRequest request = buildAlertRequest(guid, comment, username, timestamp);
- getDao().addCommentToAlert(request);
+ return getDao().addCommentToAlert(request);
}
- protected void removeAlertComment(String guid, String comment, String username, long timestamp)
+ protected Document removeAlertComment(String guid, String comment, String username, long timestamp)
throws IOException {
CommentAddRemoveRequest request = buildAlertRequest(guid, comment, username, timestamp);
- getDao().removeCommentFromAlert(request);
+ return getDao().removeCommentFromAlert(request);
}
private CommentAddRemoveRequest buildAlertRequest(String guid, String comment, String username,
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
index f754b81..7e28853 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
@@ -278,11 +278,40 @@ public abstract class MetaAlertIntegrationTest {
}});
setGroups(Collections.singletonList("group"));
}};
- MetaAlertCreateResponse metaAlertCreateResponse = metaDao
+
+ Document actualMetaAlert = metaDao
.createMetaAlert(metaAlertCreateRequest);
+
+ // Build expected metaAlert after alerts are added
+ Map<String, Object> expectedMetaAlert = new HashMap<>();
+
+ expectedMetaAlert.put(Constants.GUID, actualMetaAlert.getGuid());
+ expectedMetaAlert.put(getSourceTypeField(), METAALERT_TYPE);
+ expectedMetaAlert.put(STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString());
+ // Verify the proper alerts were added
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> metaAlertAlerts = new ArrayList<>();
+ // Alert 0 is already in the metaalert. Add alerts 1 and 2.
+ Map<String, Object> expectedAlert1 = alerts.get(1);
+ expectedAlert1.put(METAALERT_FIELD, Collections.singletonList(actualMetaAlert.getGuid()));
+ metaAlertAlerts.add(expectedAlert1);
+ Map<String, Object> expectedAlert2 = alerts.get(2);
+ expectedAlert2.put(METAALERT_FIELD, Collections.singletonList(actualMetaAlert.getGuid()));
+ metaAlertAlerts.add(expectedAlert2);
+ expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts);
+
+ // Verify the counts were properly updated
+ expectedMetaAlert.put("average", 1.5d);
+ expectedMetaAlert.put("min", 1.0d);
+ expectedMetaAlert.put("median", 1.5d);
+ expectedMetaAlert.put("max", 2.0d);
+ expectedMetaAlert.put("count", 2);
+ expectedMetaAlert.put("sum", 3.0d);
+ expectedMetaAlert.put(getThreatTriageField(), 3.0d);
{
// Verify metaAlert was created
- findCreatedDoc(metaAlertCreateResponse.getGuid(), METAALERT_TYPE);
+ assertEquals(expectedMetaAlert, actualMetaAlert.getDocument());
+ findCreatedDoc(actualMetaAlert.getGuid(), METAALERT_TYPE);
}
{
// Verify alert 0 was not updated with metaalert field
@@ -294,14 +323,14 @@ public abstract class MetaAlertIntegrationTest {
// Verify alert 1 was properly updated with metaalert field
Map<String, Object> expectedAlert = new HashMap<>(alerts.get(1));
expectedAlert
- .put(METAALERT_FIELD, Collections.singletonList(metaAlertCreateResponse.getGuid()));
+ .put(METAALERT_FIELD, Collections.singletonList(actualMetaAlert.getGuid()));
findUpdatedDoc(expectedAlert, "message_1", SENSOR_NAME);
}
{
// Verify alert 2 was properly updated with metaalert field
Map<String, Object> expectedAlert = new HashMap<>(alerts.get(2));
expectedAlert
- .put(METAALERT_FIELD, Collections.singletonList(metaAlertCreateResponse.getGuid()));
+ .put(METAALERT_FIELD, Collections.singletonList(actualMetaAlert.getGuid()));
findUpdatedDoc(expectedAlert, "message_2", SENSOR_NAME);
}
}
@@ -355,17 +384,19 @@ public abstract class MetaAlertIntegrationTest {
{
// Verify alerts were successfully added to the meta alert
- Assert.assertTrue(metaDao.addAlertsToMetaAlert("meta_alert", Arrays
- .asList(new GetRequest("message_1", SENSOR_NAME),
- new GetRequest("message_2", SENSOR_NAME))));
+ Document actualMetaAlert = metaDao.addAlertsToMetaAlert("meta_alert", Arrays
+ .asList(new GetRequest("message_1", SENSOR_NAME),
+ new GetRequest("message_2", SENSOR_NAME)));
+ assertEquals(expectedMetaAlert, actualMetaAlert.getDocument());
findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
}
{
// Verify False when alerts are already in a meta alert and no new alerts are added
- Assert.assertFalse(metaDao.addAlertsToMetaAlert("meta_alert", Arrays
- .asList(new GetRequest("message_0", SENSOR_NAME),
- new GetRequest("message_1", SENSOR_NAME))));
+ Document actualMetaAlert = metaDao.addAlertsToMetaAlert("meta_alert", Arrays
+ .asList(new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_1", SENSOR_NAME)));
+ assertEquals(expectedMetaAlert, actualMetaAlert.getDocument());
findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
}
@@ -385,9 +416,10 @@ public abstract class MetaAlertIntegrationTest {
expectedMetaAlert.put("sum", 6.0d);
expectedMetaAlert.put(getThreatTriageField(), 6.0d);
- Assert.assertTrue(metaDao.addAlertsToMetaAlert("meta_alert", Arrays
- .asList(new GetRequest("message_2", SENSOR_NAME),
- new GetRequest("message_3", SENSOR_NAME))));
+ Document actualMetaAlert = metaDao.addAlertsToMetaAlert("meta_alert", Arrays
+ .asList(new GetRequest("message_2", SENSOR_NAME),
+ new GetRequest("message_3", SENSOR_NAME)));
+ assertEquals(expectedMetaAlert, actualMetaAlert.getDocument());
findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
}
}
@@ -437,17 +469,19 @@ public abstract class MetaAlertIntegrationTest {
{
// Verify a list of alerts are removed from a meta alert
- Assert.assertTrue(metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays
- .asList(new GetRequest("message_0", SENSOR_NAME),
- new GetRequest("message_1", SENSOR_NAME))));
+ Document actualMetaAlert = metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays
+ .asList(new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_1", SENSOR_NAME)));
+ assertEquals(expectedMetaAlert, actualMetaAlert.getDocument());
findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
}
{
// Verify False when alerts are not present in a meta alert and no alerts are removed
- Assert.assertFalse(metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays
- .asList(new GetRequest("message_0", SENSOR_NAME),
- new GetRequest("message_1", SENSOR_NAME))));
+ Document actualMetaAlert = metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays
+ .asList(new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_1", SENSOR_NAME)));
+ assertEquals(expectedMetaAlert, actualMetaAlert.getDocument());
findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
}
@@ -466,9 +500,10 @@ public abstract class MetaAlertIntegrationTest {
expectedMetaAlert.put("sum", 3.0d);
expectedMetaAlert.put(getThreatTriageField(), 3.0d);
- Assert.assertTrue(metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays
- .asList(new GetRequest("message_0", SENSOR_NAME),
- new GetRequest("message_2", SENSOR_NAME))));
+ Document actualMetaAlert = metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays
+ .asList(new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_2", SENSOR_NAME)));
+ assertEquals(expectedMetaAlert, actualMetaAlert.getDocument());
findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
}
@@ -587,11 +622,11 @@ public abstract class MetaAlertIntegrationTest {
{
// Verify status changed to inactive and child alerts are updated
- Assert.assertTrue(metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.INACTIVE));
-
Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert);
expectedMetaAlert.put(STATUS_FIELD, MetaAlertStatus.INACTIVE.getStatusString());
+ Document actualMetaAlert = metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.INACTIVE);
+ Assert.assertEquals(expectedMetaAlert, actualMetaAlert.getDocument());
findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
for (int i = 0; i < numChildAlerts; ++i) {
@@ -610,11 +645,11 @@ public abstract class MetaAlertIntegrationTest {
{
// Verify status changed to active and child alerts are updated
- Assert.assertTrue(metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.ACTIVE));
-
Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert);
expectedMetaAlert.put(STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString());
+ Document actualMetaAlert = metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.ACTIVE);
+ Assert.assertEquals(expectedMetaAlert, actualMetaAlert.getDocument());
findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
for (int i = 0; i < numChildAlerts; ++i) {
@@ -629,11 +664,15 @@ public abstract class MetaAlertIntegrationTest {
// Make sure to handle the guid offset from creation
findUpdatedDoc(expectedAlert, "message_" + (i + numChildAlerts), SENSOR_NAME);
}
-
+ }
+ {
{
// Verify status changed to current status has no effect
- Assert.assertFalse(metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.ACTIVE));
+ Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert);
+ expectedMetaAlert.put(STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString());
+ Document actualMetaAlert = metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.ACTIVE);
+ Assert.assertEquals(expectedMetaAlert, actualMetaAlert.getDocument());
findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
for (int i = 0; i < numChildAlerts; ++i) {
@@ -985,6 +1024,30 @@ public abstract class MetaAlertIntegrationTest {
throw new OriginalNotFoundException("Count not find guids after " + MAX_RETRIES + "tries");
}
+ @SuppressWarnings("unchecked")
+ protected void assertEquals(Map<String, Object> expected, Map<String, Object> actual) {
+ Assert.assertEquals(expected.get(Constants.GUID), actual.get(Constants.GUID));
+ Assert.assertEquals(expected.get(getSourceTypeField()), actual.get(getSourceTypeField()));
+ Double actualThreatTriageField = actual.get(getThreatTriageField()) instanceof Float ?
+ ((Float) actual.get(getThreatTriageField())).doubleValue() : (Double) actual.get(getThreatTriageField());
+ Assert.assertEquals(expected.get(getThreatTriageField()), actualThreatTriageField);
+
+ List<Map<String, Object>> expectedAlerts = (List<Map<String, Object>>) expected.get(ALERT_FIELD);
+ List<Map<String, Object>> actualAlerts = (List<Map<String, Object>>) actual.get(ALERT_FIELD);
+ expectedAlerts.sort(Comparator.comparing(o -> ((String) o.get(Constants.GUID))));
+ actualAlerts.sort(Comparator.comparing(o -> ((String) o.get(Constants.GUID))));
+ Assert.assertEquals(expectedAlerts, actualAlerts);
+ Assert.assertEquals(expected.get(STATUS_FIELD), actual.get(STATUS_FIELD));
+ Assert.assertEquals(expected.get("average"), actual.get("average"));
+ Assert.assertEquals(expected.get("min"), actual.get("min"));
+ Assert.assertEquals(expected.get("median"), actual.get("median"));
+ Assert.assertEquals(expected.get("max"), actual.get("max"));
+ Integer actualCountField = actual.get("count") instanceof Long ? ((Long) actual.get("count")).intValue() :
+ (Integer) actual.get("count");
+ Assert.assertEquals(expected.get("count"), actualCountField);
+ Assert.assertEquals(expected.get("sum"), actual.get("sum"));
+ }
+
protected List<Map<String, Object>> buildAlerts(int count) {
List<Map<String, Object>> inputData = new ArrayList<>();
for (int i = 0; i < count; ++i) {
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
index 5a70636..ec241e4 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
@@ -54,12 +54,10 @@ import org.apache.metron.indexing.dao.RetrieveLatestDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
-import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
import org.apache.metron.indexing.dao.metaalert.MetaAlertRetrieveLatestDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
import org.apache.metron.indexing.dao.metaalert.MetaScores;
import org.apache.metron.indexing.dao.search.GetRequest;
-import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.PatchRequest;
@@ -157,43 +155,39 @@ public class AbstractLuceneMetaAlertUpdateDaoTest {
}
@Override
- public void update(Document update, Optional<String> index) {
+ public Document update(Document update, Optional<String> index) {
+ return null;
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) {
+ public Document addCommentToAlert(CommentAddRemoveRequest request) {
+ return null;
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request) {
+ return null;
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ return null;
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ return null;
}
@Override
- public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
+ public Document patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
Optional<Long> timestamp) {
- }
-
- @Override
- public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request) {
return null;
}
@Override
- public boolean addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests) {
- return false;
- }
-
- @Override
- public boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status) {
- return false;
+ public Document createMetaAlert(MetaAlertCreateRequest request) {
+ return null;
}
}
@@ -762,6 +756,30 @@ public class AbstractLuceneMetaAlertUpdateDaoTest {
UUID.fromString((String) actualDocument.get(Constants.GUID));
}
+ @Test
+ public void addAlertsToMetaAlertShouldThrowExceptionOnMissingMetaAlert() throws Exception {
+ thrown.expect(IOException.class);
+ thrown.expectMessage("Unable to add alerts to meta alert. Meta alert with guid some_guid cannot be found.");
+
+ dao.addAlertsToMetaAlert("some_guid", new ArrayList<>());
+ }
+
+ @Test
+ public void removeAlertsFromMetaAlertShouldThrowExceptionOnMissingMetaAlert() throws Exception {
+ thrown.expect(IOException.class);
+ thrown.expectMessage("Unable to remove alerts from meta alert. Meta alert with guid some_guid cannot be found.");
+
+ dao.removeAlertsFromMetaAlert("some_guid", new ArrayList<>());
+ }
+
+ @Test
+ public void updateMetaAlertStatusShouldThrowExceptionOnMissingMetaAlert() throws Exception {
+ thrown.expect(IOException.class);
+ thrown.expectMessage("Unable to update meta alert status. Meta alert with guid some_guid cannot be found.");
+
+ dao.updateMetaAlertStatus("some_guid", MetaAlertStatus.INACTIVE);
+ }
+
// Utility method to manage comparing update maps
protected boolean updatesMapEquals(Map<Document, Optional<String>> expected,
Map<Document, Optional<String>> actual) {
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
index 73a9077..8eaa8f4 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
@@ -35,14 +35,13 @@ import org.apache.metron.hbase.mock.MockHTable;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.HBaseDao;
import org.apache.metron.indexing.dao.IndexDao;
-import org.apache.metron.indexing.dao.MultiIndexDao;
import org.apache.metron.indexing.dao.UpdateIntegrationTest;
import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.junit.After;
import org.junit.Assert;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.Test;
public class HBaseDaoIntegrationTest extends UpdateIntegrationTest {
@@ -61,8 +60,8 @@ public class HBaseDaoIntegrationTest extends UpdateIntegrationTest {
0x54,0x79,0x70,0x65
};
- @BeforeClass
- public static void startHBase() throws Exception {
+ @Before
+ public void startHBase() throws Exception {
AccessConfig accessConfig = new AccessConfig();
accessConfig.setMaxSearchResults(1000);
accessConfig.setMaxSearchGroups(1000);
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
index a840bb4..e6906f3 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
@@ -124,30 +124,30 @@ public class SolrDao implements IndexDao {
}
@Override
- public void update(Document update, Optional<String> index) throws IOException {
- this.solrUpdateDao.update(update, index);
+ public Document update(Document update, Optional<String> index) throws IOException {
+ return this.solrUpdateDao.update(update, index);
}
@Override
- public void batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
- this.solrUpdateDao.batchUpdate(updates);
+ public Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
+ return this.solrUpdateDao.batchUpdate(updates);
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
- this.solrUpdateDao.addCommentToAlert(request);
+ public Document addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ return this.solrUpdateDao.addCommentToAlert(request);
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
- this.solrUpdateDao.removeCommentFromAlert(request);
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ return this.solrUpdateDao.removeCommentFromAlert(request);
}
@Override
- public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
+ public Document patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
Optional<Long> timestamp)
throws OriginalNotFoundException, IOException {
- solrUpdateDao.patch(retrieveLatestDao, request, timestamp);
+ return solrUpdateDao.patch(retrieveLatestDao, request, timestamp);
}
@Override
@@ -156,15 +156,15 @@ public class SolrDao implements IndexDao {
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest)
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest)
throws IOException {
- this.solrUpdateDao.addCommentToAlert(request, latest);
+ return this.solrUpdateDao.addCommentToAlert(request, latest);
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
throws IOException {
- this.solrUpdateDao.removeCommentFromAlert(request, latest);
+ return this.solrUpdateDao.removeCommentFromAlert(request, latest);
}
/**
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
index 4748315..8ef9484 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
@@ -33,7 +33,6 @@ import org.apache.metron.indexing.dao.RetrieveLatestDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
-import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
import org.apache.metron.indexing.dao.metaalert.MetaAlertDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
import org.apache.metron.indexing.dao.search.FieldType;
@@ -190,20 +189,20 @@ public class SolrMetaAlertDao implements MetaAlertDao {
}
@Override
- public void update(Document update, Optional<String> index) throws IOException {
- metaAlertUpdateDao.update(update, index);
+ public Document update(Document update, Optional<String> index) throws IOException {
+ return metaAlertUpdateDao.update(update, index);
}
@Override
- public void batchUpdate(Map<Document, Optional<String>> updates) {
- metaAlertUpdateDao.batchUpdate(updates);
+ public Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) {
+ return metaAlertUpdateDao.batchUpdate(updates);
}
@Override
- public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
+ public Document patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
Optional<Long> timestamp)
throws OriginalNotFoundException, IOException {
- metaAlertUpdateDao.patch(retrieveLatestDao, request, timestamp);
+ return metaAlertUpdateDao.patch(retrieveLatestDao, request, timestamp);
}
@Override
@@ -212,46 +211,46 @@ public class SolrMetaAlertDao implements MetaAlertDao {
}
@Override
- public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request)
+ public Document createMetaAlert(MetaAlertCreateRequest request)
throws InvalidCreateException, IOException {
return metaAlertUpdateDao.createMetaAlert(request);
}
@Override
- public boolean addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
+ public Document addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
throws IOException {
return metaAlertUpdateDao.addAlertsToMetaAlert(metaAlertGuid, alertRequests);
}
@Override
- public boolean removeAlertsFromMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
+ public Document removeAlertsFromMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
throws IOException {
return metaAlertUpdateDao.removeAlertsFromMetaAlert(metaAlertGuid, alertRequests);
}
@Override
- public boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status)
+ public Document updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status)
throws IOException {
return metaAlertUpdateDao.updateMetaAlertStatus(metaAlertGuid, status);
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
- solrDao.addCommentToAlert(request);
+ public Document addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ return solrDao.addCommentToAlert(request);
}
- @Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
- solrDao.removeCommentFromAlert(request);
- }
+ @Override
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ return solrDao.removeCommentFromAlert(request);
+ }
- @Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
- solrDao.addCommentToAlert(request, latest);
- }
+ @Override
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ return solrDao.addCommentToAlert(request, latest);
+ }
- @Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
- solrDao.removeCommentFromAlert(request, latest);
- }
+ @Override
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ return solrDao.removeCommentFromAlert(request, latest);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
index 132d872..124d4be 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
@@ -28,11 +28,9 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
-import org.apache.metron.common.Constants;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
-import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
import org.apache.metron.indexing.dao.metaalert.MetaAlertUpdateDao;
import org.apache.metron.indexing.dao.metaalert.MetaScores;
@@ -71,7 +69,7 @@ public class SolrMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao imp
}
@Override
- public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request)
+ public Document createMetaAlert(MetaAlertCreateRequest request)
throws InvalidCreateException, IOException {
List<GetRequest> alertRequests = request.getAlerts();
if (request.getAlerts().isEmpty()) {
@@ -120,11 +118,8 @@ public class SolrMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao imp
// Kick off any updates.
update(updates);
- MetaAlertCreateResponse createResponse = new MetaAlertCreateResponse();
- createResponse.setCreated(true);
- createResponse.setGuid(metaAlert.getGuid());
solrClient.commit(METAALERTS_COLLECTION);
- return createResponse;
+ return metaAlert;
} catch (IOException | SolrServerException e) {
throw new InvalidCreateException("Unable to create meta alert", e);
}
@@ -135,10 +130,11 @@ public class SolrMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao imp
* Updates a document in Solr for a given collection. Collection is not optional for Solr.
* @param update The update to be run
* @param collection The index to be updated. Mandatory for Solr
+ * @return The updated document.
* @throws IOException Thrown when an error occurs during the write.
*/
@Override
- public void update(Document update, Optional<String> collection) throws IOException {
+ public Document update(Document update, Optional<String> collection) throws IOException {
if (MetaAlertConstants.METAALERT_TYPE.equals(update.getSensorType())) {
// We've been passed an update to the meta alert.
throw new UnsupportedOperationException("Meta alerts cannot be directly updated");
@@ -181,28 +177,30 @@ public class SolrMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao imp
} catch (SolrServerException e) {
throw new IOException("Unable to update document", e);
}
+
+ return update;
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
- getUpdateDao().addCommentToAlert(request);
+ public Document addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ return getUpdateDao().addCommentToAlert(request);
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
- getUpdateDao().removeCommentFromAlert(request);
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ return getUpdateDao().removeCommentFromAlert(request);
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest)
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest)
throws IOException {
- getUpdateDao().addCommentToAlert(request, latest);
+ return getUpdateDao().addCommentToAlert(request, latest);
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
throws IOException {
- getUpdateDao().removeCommentFromAlert(request, latest);
+ return getUpdateDao().removeCommentFromAlert(request, latest);
}
protected boolean replaceAlertInMetaAlert(Document metaAlert, Document alert) {
@@ -215,9 +213,8 @@ public class SolrMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao imp
}
@Override
- public boolean addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
- throws IOException {
- boolean success;
+ public Document addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
+ throws IOException, IllegalStateException {
Document metaAlert = getRetrieveLatestDao()
.getLatest(metaAlertGuid, MetaAlertConstants.METAALERT_TYPE);
if (MetaAlertStatus.ACTIVE.getStatusString()
@@ -225,7 +222,6 @@ public class SolrMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao imp
Iterable<Document> alerts = getRetrieveLatestDao().getAllLatest(alertRequests);
Map<Document, Optional<String>> updates = buildAddAlertToMetaAlertUpdates(metaAlert, alerts);
update(updates);
- success = updates.size() != 0;
} else {
throw new IllegalStateException("Adding alerts to an INACTIVE meta alert is not allowed");
}
@@ -234,6 +230,6 @@ public class SolrMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao imp
} catch (SolrServerException e) {
throw new IOException("Unable to commit alerts to metaalert: " + metaAlertGuid, e);
}
- return success;
+ return metaAlert;
}
}
[51/51] [abbrv] metron git commit: Revert "METRON-1801 Allow
Customization of Elasticsearch Document ID (nickwallen) closes
apache/metron#1218"
Posted by ot...@apache.org.
Revert "METRON-1801 Allow Customization of Elasticsearch Document ID (nickwallen) closes apache/metron#1218"
This reverts commit 90c5e1d2896b8a95bd6c928e3dab96f1fd85167c.
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0e037eda
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0e037eda
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0e037eda
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 0e037edad913955d3b6754ca9cf42b329cd84160
Parents: 0ba1802
Author: Nick Allen <ni...@nickallen.org>
Authored: Tue Oct 23 15:01:04 2018 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Tue Oct 23 15:01:04 2018 -0400
----------------------------------------------------------------------
Upgrading.md | 23 +--
.../METRON/CURRENT/configuration/metron-env.xml | 9 --
.../CURRENT/package/scripts/metron_service.py | 5 -
.../package/scripts/params/params_linux.py | 1 -
.../METRON/CURRENT/themes/metron_theme.json | 10 --
metron-platform/metron-elasticsearch/README.md | 37 +----
metron-platform/metron-elasticsearch/pom.xml | 17 +++
.../dao/ElasticsearchRetrieveLatestDao.java | 147 +++++++------------
.../dao/ElasticsearchSearchDao.java | 28 +---
.../elasticsearch/utils/ElasticsearchUtils.java | 100 ++++---------
.../writer/ElasticsearchWriter.java | 39 ++---
.../elasticsearch/dao/ElasticsearchDaoTest.java | 10 +-
.../ElasticsearchSearchIntegrationTest.java | 20 +--
.../indexing/dao/SearchIntegrationTest.java | 21 ++-
.../integration/SolrSearchIntegrationTest.java | 19 ---
15 files changed, 145 insertions(+), 341 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/Upgrading.md
----------------------------------------------------------------------
diff --git a/Upgrading.md b/Upgrading.md
index c3b2a0f..2124ac5 100644
--- a/Upgrading.md
+++ b/Upgrading.md
@@ -16,22 +16,9 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
# Upgrading
-
This document constitutes a per-version listing of changes of
configuration which are non-backwards compatible.
-## 0.6.0 to 0.6.1
-
-### [METRON-1801 Allow Customization of Elasticsearch Document ID](https://issues.apache.org/jira/browse/METRON-1801)
-
-A global property named `es.document.id` was added to define the field from which the document ID is set when a message is indexed by Elasticsearch. To allow Elasticsearch to define its own document id, this property should be set to a blank or empty string. The client will not set the document ID and Elasticsearch will define its own. In most cases allowing Elasticsearch to define the document ID is the most performant option. This is now the default behavior.
-
-Metron versions 0.6.0 and earlier defined the document ID using the Metron GUID, which is a randomized UUID using Java's `UUID.randomUUID()`. Using a randomized UUID can negatively impact Elasticsearch indexing performance. To maintain backwards compatibility with legacy versions of Metron use the following global property setting.
-
- ```
- es.document.id = guid
- ```
-
## 0.4.2 to 0.5.0
### [METRON-941: native PaloAlto parser corrupts message when having a comma in the payload](https://issues.apache.org/jira/browse/METRON-941)
@@ -102,7 +89,7 @@ For a more detailed description, please see metron-platform/metron-elasticsearch
### Description
-In the 0.4.2 release,
+In the 0.4.2 release,
## 0.3.1 to 0.4.0
@@ -120,7 +107,7 @@ This effectively limits the build environment to Docker supported [platforms](ht
#### Description
As of 0.3.0 the indexing configuration
-* Is held in the enrichment configuration for a sensor
+* Is held in the enrichment configuration for a sensor
* Has properties which control every writers (i.e. HDFS, solr or elasticsearch).
In the 0.3.1 release, this configuration has been broken out
@@ -149,7 +136,7 @@ You would create a file to configure each writer for sensor `foo` called `$METRO
"batchSize" : 100,
"enabled" : true
},
- "hdfs" : {
+ "hdfs" : {
"index" : "foo",
"batchSize" : 100,
"enabled" : true
@@ -164,7 +151,7 @@ You would create a file to configure each writer for sensor `foo` called `$METRO
As of 0.3.0, threat triage rules were defined as a simple Map associating a Stellar expression with a score.
As of 0.3.1, due to the fact that there may be many threat triage rules, we have made the rules more complex.
To help organize these, we have made the threat triage objects in their own right that contain optional name and optional comment fields.
-
+
This essentially makes the risk level rules slightly more complex. The format goes from:
```
"riskLevelRules" : {
@@ -182,7 +169,7 @@ to:
}
]
```
-
+
#### Migration
For every sensor enrichment configuration, you will need to migrate the `riskLevelRules` section
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index 0c7d295..81dda6c 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -122,15 +122,6 @@
</value-attributes>
</property>
<property>
- <name>es_document_id</name>
- <description>Message field containing the Elasticsearch document ID</description>
- <value></value>
- <display-name>Elasticsearch Document ID Source Field</display-name>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- </property>
- <property>
<name>solr_zookeeper_url</name>
<value>{{zookeeper_quorum}}</value>
<description>Comma delimited list of Zookeeper Urls: zkhost1:2181,zkhost1:2181</description>
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
index 9b9d99b..9d15e93 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
@@ -83,11 +83,6 @@ def elasticsearch_global_config_patches():
"op": "add",
"path": "/es.date.format",
"value": "{{es_date_format}}"
- },
- {
- "op": "add",
- "path": "/es.document.id",
- "value": "{{es_document_id}}"
}
"""
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 316964a..458a7be 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -101,7 +101,6 @@ es_url = ",".join([host + ":" + es_binary_port for host in es_host_list])
es_http_port = config['configurations']['metron-env']['es_http_port']
es_http_url = es_host_list[0] + ":" + es_http_port
es_date_format = config['configurations']['metron-env']['es_date_format']
-es_document_id = config['configurations']['metron-env']['es_document_id']
# hadoop params
stack_root = Script.get_stack_root()
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index 34eff50..7e6c83a 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -476,10 +476,6 @@
"subsection-name": "subsection-index-settings"
},
{
- "config": "metron-env/es_document_id",
- "subsection-name": "subsection-index-settings"
- },
- {
"config": "metron-env/solr_zookeeper_url",
"subsection-name": "subsection-index-settings"
},
@@ -953,12 +949,6 @@
}
},
{
- "config": "metron-env/es_document_id",
- "widget": {
- "type": "text-field"
- }
- },
- {
"config": "metron-env/solr_zookeeper_url",
"widget": {
"type": "text-field"
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/metron-platform/metron-elasticsearch/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/README.md b/metron-platform/metron-elasticsearch/README.md
index 25e15b4..177412e 100644
--- a/metron-platform/metron-elasticsearch/README.md
+++ b/metron-platform/metron-elasticsearch/README.md
@@ -59,39 +59,6 @@ For instance, an `es.date.format` of `yyyy.MM.dd.HH` would have the consequence
roll hourly, whereas an `es.date.format` of `yyyy.MM.dd` would have the consequence that the indices would
roll daily.
-### `es.document.id`
-
-This property sets the message field that is used to define the document ID when a message is indexed by Elasticsearch. By default, the client does not set the document ID and document ID generation is deferred to Elasticsearch.
-
-#### Option 1: Defer to Elasticsearch
-
-Value: (Undefined, blank or empty string)
-
-* This option allows Elasticsearch to generate the document ID.
-* The client will not set a document ID.
-* In most cases this is the most performant option.
-* This is the default behavior.
-
-#### Option 2: Legacy Compatibility
-
-Value: guid
-
-* Metron versions 0.6.0 and earlier defined the document ID using the Metron GUID, which is a randomized UUID using Java's `UUID.randomUUID()`.
-* Using a randomized UUID can negatively impact Elasticsearch indexing performance.
-* To maintain backwards compatibility with legacy versions of Metron, set the value to `guid`.
- ```
- es.document.id = guid
- ```
-
-#### Option 3: Custom Document ID
-
-* Advanced users can define a custom document ID.
-* Create an enrichment that defines a new message field; for example one called `my_document_id`. Use this field to set the document ID. This will set the document ID to the value of the message field `my_document_id`.
- ```
- es.document.id = my_document_id
- ```
-* If a message does not contain the `es.document.id` field, a warning is issued and no document ID is set by the client.
-
## Upgrading to 5.6.2
Users should be prepared to re-index when migrating from Elasticsearch 2.3.3 to 5.6.2. There are a number of template changes, most notably around
@@ -354,7 +321,7 @@ We'll want to put the template back into Elasticsearch:
curl -XPUT "http://${ELASTICSEARCH}:9200/_template/${SENSOR}_index" -d @${SENSOR}.template
```
-To update existing indexes, update Elasticsearch mappings with the new field for each sensor.
+To update existing indexes, update Elasticsearch mappings with the new field for each sensor.
```
curl -XPUT "http://${ELASTICSEARCH}:9200/${SENSOR}_index*/_mapping/${SENSOR}_doc" -d '
@@ -373,7 +340,7 @@ rm ${SENSOR}.template
The stock set of Elasticsearch templates for bro, snort, yaf, error index and meta index are installed automatically during the first time install and startup of Metron Indexing service.
-It is possible that Elasticsearch service is not available when the Metron Indexing Service startup, in that case the Elasticsearch template will not be installed.
+It is possible that Elasticsearch service is not available when the Metron Indexing Service startup, in that case the Elasticsearch template will not be installed.
For such a scenario, an Admin can have the template installed in two ways:
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/metron-platform/metron-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index edec1bb..adc601a 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -206,7 +206,24 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>${global_log4j_core_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${global_log4j_core_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava-testlib</artifactId>
+ <version>${global_guava_version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
+
<build>
<plugins>
<plugin>
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRetrieveLatestDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRetrieveLatestDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRetrieveLatestDao.java
index 2f00b68..f6bfeda 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRetrieveLatestDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRetrieveLatestDao.java
@@ -20,22 +20,7 @@ package org.apache.metron.elasticsearch.dao;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.metron.common.Constants;
-import org.apache.metron.elasticsearch.utils.ElasticsearchUtils;
-import org.apache.metron.indexing.dao.RetrieveLatestDao;
-import org.apache.metron.indexing.dao.search.GetRequest;
-import org.apache.metron.indexing.dao.update.Document;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHits;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.IOException;
-import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -43,13 +28,19 @@ import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
-
-import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
+import org.apache.metron.indexing.dao.RetrieveLatestDao;
+import org.apache.metron.indexing.dao.search.GetRequest;
+import org.apache.metron.indexing.dao.update.Document;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.index.query.IdsQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
public class ElasticsearchRetrieveLatestDao implements RetrieveLatestDao {
- protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private TransportClient transportClient;
public ElasticsearchRetrieveLatestDao(TransportClient transportClient) {
@@ -58,7 +49,7 @@ public class ElasticsearchRetrieveLatestDao implements RetrieveLatestDao {
@Override
public Document getLatest(String guid, String sensorType) {
- Optional<Document> doc = searchByGuid(guid, sensorType, hit -> toDocument(hit));
+ Optional<Document> doc = searchByGuid(guid, sensorType, hit -> toDocument(guid, hit));
return doc.orElse(null);
}
@@ -70,19 +61,26 @@ public class ElasticsearchRetrieveLatestDao implements RetrieveLatestDao {
guids.add(getRequest.getGuid());
sensorTypes.add(getRequest.getSensorType());
}
- return searchByGuids(guids, sensorTypes, hit -> toDocument(hit));
+ List<Document> documents = searchByGuids(
+ guids,
+ sensorTypes,
+ hit -> {
+ Long ts = 0L;
+ String doc = hit.getSourceAsString();
+ String sourceType = Iterables.getFirst(Splitter.on("_doc").split(hit.getType()), null);
+ try {
+ return Optional.of(new Document(doc, hit.getId(), sourceType, ts));
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to retrieve latest: " + e.getMessage(), e);
+ }
+ }
+
+ );
+ return documents;
}
- /**
- * Search for a Metron GUID.
- *
- * @param guid The Metron GUID to search for.
- * @param sensorType The sensor type to include in the search.
- * @param callback A callback that transforms a search hit to a search result.
- * @param <T> The type of search results expected.
- * @return All documents with the given Metron GUID and sensor type.
- */
- <T> Optional<T> searchByGuid(String guid, String sensorType, Function<SearchHit, Optional<T>> callback) {
+ <T> Optional<T> searchByGuid(String guid, String sensorType,
+ Function<SearchHit, Optional<T>> callback) {
Collection<String> sensorTypes = sensorType != null ? Collections.singleton(sensorType) : null;
List<T> results = searchByGuids(Collections.singleton(guid), sensorTypes, callback);
if (results.size() > 0) {
@@ -93,69 +91,48 @@ public class ElasticsearchRetrieveLatestDao implements RetrieveLatestDao {
}
/**
- * Search for one or more Metron GUIDs.
- *
- * @param guids The Metron GUIDs to search by.
- * @param sensorTypes The sensor types to include in the search.
- * @param callback A callback that transforms a search hit to a search result.
- * @param <T> The type of expected results.
- * @return All documents with the given Metron GUID and sensor type.
+ * Return the search hit based on the UUID and sensor type.
+ * A callback can be specified to transform the hit into a type T.
+ * If more than one hit happens, the first one will be returned.
*/
- <T> List<T> searchByGuids(Collection<String> guids,
- Collection<String> sensorTypes,
- Function<SearchHit, Optional<T>> callback) {
+ <T> List<T> searchByGuids(Collection<String> guids, Collection<String> sensorTypes,
+ Function<SearchHit, Optional<T>> callback) {
if (guids == null || guids.isEmpty()) {
return Collections.emptyList();
}
-
- // search by metron's GUID field
- QueryBuilder query = boolQuery().must(termsQuery(Constants.GUID, guids));
- SearchRequestBuilder request = transportClient
- .prepareSearch()
- .setQuery(query)
- .setSize(guids.size());
-
- if(sensorTypes != null) {
- request.setTypes(toDocTypes(sensorTypes));
+ QueryBuilder query = null;
+ IdsQueryBuilder idsQuery;
+ if (sensorTypes != null) {
+ String[] types = sensorTypes.stream().map(sensorType -> sensorType + "_doc")
+ .toArray(String[]::new);
+ idsQuery = QueryBuilders.idsQuery(types);
+ } else {
+ idsQuery = QueryBuilders.idsQuery();
}
- // transform the search hits to results
- SearchHits hits = request.get().getHits();
- return getResults(hits, callback);
- }
+ for (String guid : guids) {
+ query = idsQuery.addIds(guid);
+ }
- /**
- * Retrieve search results.
- *
- * @param hits The search hit.
- * @param callback A callback function that transforms search hits to search results.
- * @param <T> The execpted type of search result.
- * @return The search results.
- */
- private <T> List<T> getResults(SearchHits hits, Function<SearchHit, Optional<T>> callback) {
+ SearchRequestBuilder request = transportClient.prepareSearch()
+ .setQuery(query)
+ .setSize(guids.size());
+ org.elasticsearch.action.search.SearchResponse response = request.get();
+ SearchHits hits = response.getHits();
List<T> results = new ArrayList<>();
-
for (SearchHit hit : hits) {
Optional<T> result = callback.apply(hit);
if (result.isPresent()) {
results.add(result.get());
}
}
-
return results;
}
- /**
- * Transforms a {@link SearchHit} to a {@link Document}.
- *
- * @param hit The result of a search.
- * @return An optional {@link Document}.
- */
- private Optional<Document> toDocument(SearchHit hit) {
+ private Optional<Document> toDocument(final String guid, SearchHit hit) {
Long ts = 0L;
String doc = hit.getSourceAsString();
String sourceType = toSourceType(hit.getType());
- String guid = ElasticsearchUtils.getGUID(hit);
try {
return Optional.of(new Document(doc, guid, sourceType, ts));
} catch (IOException e) {
@@ -171,26 +148,4 @@ public class ElasticsearchRetrieveLatestDao implements RetrieveLatestDao {
private String toSourceType(String docType) {
return Iterables.getFirst(Splitter.on("_doc").split(docType), null);
}
-
- /**
- * Returns the doc types for a given collection of sensor types.
- *
- * @param sensorTypes The sensor types.
- * @return The doc types associated with each sensor.
- */
- private String[] toDocTypes(Collection<String> sensorTypes) {
- String[] result;
-
- if(sensorTypes != null && sensorTypes.size() > 0) {
- result = sensorTypes
- .stream()
- .map(sensorType -> sensorType + "_doc")
- .toArray(String[]::new);
-
- } else {
- result = new String[0];
- }
-
- return result;
- }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
index c944578..5cd0a4d 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
@@ -33,8 +33,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
-
-import org.apache.metron.common.Constants;
import org.apache.metron.elasticsearch.utils.ElasticsearchUtils;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.search.FieldType;
@@ -76,8 +74,6 @@ import org.elasticsearch.search.sort.FieldSortBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.metron.common.Constants.GUID;
-
public class ElasticsearchSearchDao implements SearchDao {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -288,29 +284,19 @@ public class ElasticsearchSearchDao implements SearchDao {
.toArray(value -> new String[indices.size()]);
}
- /**
- * Transforms a {@link SearchHit} to a {@link SearchResult}.
- *
- * @param searchHit The result of a search request.
- * @param fieldsToKeep The source fields to keep. All others are excluded. If null, all source fields are kept.
- * @return A search result.
- */
- private SearchResult getSearchResult(SearchHit searchHit, List<String> fieldsToKeep) {
-
+ private SearchResult getSearchResult(SearchHit searchHit, List<String> fields) {
+ SearchResult searchResult = new SearchResult();
+ searchResult.setId(searchHit.getId());
Map<String, Object> source;
- if (fieldsToKeep != null) {
+ if (fields != null) {
+ Map<String, Object> resultSourceAsMap = searchHit.getSourceAsMap();
source = new HashMap<>();
- // must always keep the GUID
- fieldsToKeep.add(Constants.GUID);
- fieldsToKeep.forEach(field -> {
- source.put(field, searchHit.getSourceAsMap().get(field));
+ fields.forEach(field -> {
+ source.put(field, resultSourceAsMap.get(field));
});
} else {
source = searchHit.getSource();
}
-
- SearchResult searchResult = new SearchResult();
- searchResult.setId(ElasticsearchUtils.getGUID(searchHit));
searchResult.setSource(source);
searchResult.setScore(searchHit.getScore());
searchResult.setIndex(searchHit.getIndex());
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
index 31503fa..98dc66d 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
@@ -17,9 +17,26 @@
*/
package org.apache.metron.elasticsearch.utils;
+import static java.lang.String.format;
+
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.utils.HDFSUtils;
@@ -36,29 +53,10 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static java.lang.String.format;
-import static org.apache.metron.common.Constants.GUID;
-
public class ElasticsearchUtils {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -67,14 +65,6 @@ public class ElasticsearchUtils {
private static final String USERNAME_CONFIG_KEY = "es.xpack.username";
private static final String TRANSPORT_CLIENT_USER_KEY = "xpack.security.user";
- /**
- * Defines which message field, the document identifier is set to.
- *
- * <p>If defined, the value of the specified message field is set as the Elasticsearch doc ID. If
- * this field is undefined or blank, then the document identifier is not set.
- */
- public static final String DOC_ID_SOURCE_FIELD = "es.document.id";
- public static final String DOC_ID_SOURCE_FIELD_DEFAULT = "";
private static ThreadLocal<Map<String, SimpleDateFormat>> DATE_FORMAT_CACHE
= ThreadLocal.withInitial(() -> new HashMap<>());
@@ -383,48 +373,16 @@ public class ElasticsearchUtils {
* @param searchResponse An Elasticsearch SearchHit to be converted.
* @return The list of SearchResults for the SearchHit
*/
- protected static List<SearchResult> getSearchResults(org.elasticsearch.action.search.SearchResponse searchResponse) {
- SearchHit[] searchHits = searchResponse.getHits().getHits();
- return Arrays.stream(searchHits)
- .map(hit -> toSearchResult(hit))
- .collect(Collectors.toList());
- }
-
- /**
- * Transforms a {@link SearchHit} to a {@link SearchResult}.
- *
- * @param searchHit The search hit to transform.
- * @return A {@link SearchResult} representing the {@link SearchHit}.
- */
- protected static SearchResult toSearchResult(SearchHit searchHit) {
- SearchResult searchResult = new SearchResult();
- searchResult.setId(getGUID(searchHit));
- searchResult.setSource(searchHit.getSource());
- searchResult.setScore(searchHit.getScore());
- searchResult.setIndex(searchHit.getIndex());
- return searchResult;
- }
-
- /**
- * Retrieves the Metron GUID from a {@link SearchHit}.
- *
- * @param searchHit The search hit containing a Metron GUID.
- * @return The Metron GUID.
- */
- public static String getGUID(SearchHit searchHit) {
- String guid;
- if(searchHit.hasSource() && searchHit.getSource().containsKey(GUID)) {
- guid = (String) searchHit.getSource().get(GUID);
-
- } else if(!searchHit.hasSource()) {
- String template = "No source found, has it been disabled in the mapping? index=%s, docId=%s";
- throw new IllegalStateException(String.format(template, searchHit.getIndex(), searchHit.getId()));
-
- } else {
- String template = "Missing expected field; field=%s, index=%s, docId=%s";
- throw new IllegalStateException(String.format(template, GUID, searchHit.getIndex(), searchHit.getId()));
- }
-
- return guid;
+ protected static List<SearchResult> getSearchResults(
+ org.elasticsearch.action.search.SearchResponse searchResponse) {
+ return Arrays.stream(searchResponse.getHits().getHits()).map(searchHit -> {
+ SearchResult searchResult = new SearchResult();
+ searchResult.setId(searchHit.getId());
+ searchResult.setSource(searchHit.getSource());
+ searchResult.setScore(searchHit.getScore());
+ searchResult.setIndex(searchHit.getIndex());
+ return searchResult;
+ }
+ ).collect(Collectors.toList());
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
index 9a18e8c..4b8dd08 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
@@ -17,8 +17,6 @@
*/
package org.apache.metron.elasticsearch.writer;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.ArrayUtils;
import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.field.FieldNameConverter;
@@ -45,9 +43,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import static org.apache.metron.elasticsearch.utils.ElasticsearchUtils.DOC_ID_SOURCE_FIELD;
-import static org.apache.metron.elasticsearch.utils.ElasticsearchUtils.DOC_ID_SOURCE_FIELD_DEFAULT;
-
/**
* A {@link BulkMessageWriter} that writes messages to Elasticsearch.
*/
@@ -76,48 +71,36 @@ public class ElasticsearchWriter implements BulkMessageWriter<JSONObject>, Seria
@Override
public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception {
- // writer settings
+
+ // fetch the field name converter for this sensor type
FieldNameConverter fieldNameConverter = FieldNameConverters.create(sensorType, configurations);
- final String indexPostfix = dateFormat.format(new Date());
- final String indexName = ElasticsearchUtils.getIndexName(sensorType, indexPostfix, configurations);
- final String docType = sensorType + "_doc";
- final String docIdSourceField = (String) configurations.getGlobalConfig().getOrDefault(DOC_ID_SOURCE_FIELD, DOC_ID_SOURCE_FIELD_DEFAULT);
+ final String indexPostfix = dateFormat.format(new Date());
BulkRequestBuilder bulkRequest = client.prepareBulk();
for(JSONObject message: messages) {
- // clone the message to use as the document that will be indexed
JSONObject esDoc = new JSONObject();
for(Object k : message.keySet()){
copyField(k.toString(), message, esDoc, fieldNameConverter);
}
- IndexRequestBuilder indexRequestBuilder = client
- .prepareIndex(indexName, docType)
- .setSource(esDoc.toJSONString());
-
- // set the document identifier
- if(StringUtils.isNotBlank(docIdSourceField)) {
- String docId = (String) esDoc.get(docIdSourceField);
- if(docId != null) {
- indexRequestBuilder.setId(docId);
- } else {
- LOG.warn("Message is missing document ID source field; document ID not set; sourceField={}", docIdSourceField);
- }
+ String indexName = ElasticsearchUtils.getIndexName(sensorType, indexPostfix, configurations);
+ IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName, sensorType + "_doc");
+ indexRequestBuilder = indexRequestBuilder.setSource(esDoc.toJSONString());
+ String guid = (String)esDoc.get(Constants.GUID);
+ if(guid != null) {
+ indexRequestBuilder.setId(guid);
}
- // set the document timestamp, if one exists
- Object ts = esDoc.get(Constants.Fields.TIMESTAMP.getName());
+ Object ts = esDoc.get("timestamp");
if(ts != null) {
- indexRequestBuilder.setTimestamp(ts.toString());
+ indexRequestBuilder = indexRequestBuilder.setTimestamp(ts.toString());
}
bulkRequest.add(indexRequestBuilder);
}
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
- LOG.info("Wrote {} message(s) to Elasticsearch; sensorType={}, index={}, docType={}, took={}",
- ArrayUtils.getLength(bulkResponse.getItems()), sensorType, indexName, docType, bulkResponse.getTook().format());
return buildWriteReponse(tuples, bulkResponse);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
index a2c8de5..6c3c327 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
@@ -29,8 +29,6 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
-import org.apache.metron.common.Constants;
import org.apache.metron.elasticsearch.utils.ElasticsearchUtils;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.search.FieldType;
@@ -59,20 +57,16 @@ public class ElasticsearchDaoTest {
// setup the mock search hits
SearchHit hit1 = mock(SearchHit.class);
- when(hit1.getId()).thenReturn("docId1");
- when(hit1.hasSource()).thenReturn(true);
+ when(hit1.getId()).thenReturn("id1");
when(hit1.getSource()).thenReturn(new HashMap<String, Object>() {{
put("field", "value1");
- put(Constants.GUID, "id1");
}});
when(hit1.getScore()).thenReturn(0.1f);
SearchHit hit2 = mock(SearchHit.class);
- when(hit2.getId()).thenReturn("docId2");
- when(hit2.hasSource()).thenReturn(true);
+ when(hit2.getId()).thenReturn("id2");
when(hit2.getSource()).thenReturn(new HashMap<String, Object>() {{
put("field", "value2");
- put(Constants.GUID, "id2");
}});
when(hit2.getScore()).thenReturn(0.2f);
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index df41cba..1d2d48e 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -168,6 +168,8 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
dao.search(request);
}
+
+
@Override
public void returns_column_metadata_for_specified_indices() throws Exception {
// getColumnMetadata with only bro
@@ -259,24 +261,6 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
Assert.assertEquals("data 1", results.get(0).getSource().get("ttl"));
}
- @Test
- public void queries_fields() throws Exception {
- SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class);
- SearchResponse response = getIndexDao().search(request);
- Assert.assertEquals(10, response.getTotal());
-
- List<SearchResult> results = response.getResults();
- Assert.assertEquals(10, response.getResults().size());
-
- // validate the source fields contained in the search response
- for (int i = 0; i < 10; ++i) {
- Map<String, Object> source = results.get(i).getSource();
- Assert.assertNotNull(source);
- Assert.assertNotNull(source.get(Constants.Fields.SRC_ADDR.getName()));
- Assert.assertNotNull(source.get(Constants.GUID));
- }
- }
-
@Override
protected String getSourceTypeField() {
return Constants.SENSOR_TYPE.replace('.', ':');
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index 43e8c85..cfe5752 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.adrianwalker.multilinestring.Multiline;
-import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
@@ -898,6 +897,24 @@ public abstract class SearchIntegrationTest {
}
@Test
+ public void queries_fields() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class);
+ SearchResponse response = getIndexDao().search(request);
+ Assert.assertEquals(10, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ for (int i = 0; i < 5; ++i) {
+ Map<String, Object> source = results.get(i).getSource();
+ Assert.assertEquals(1, source.size());
+ Assert.assertNotNull(source.get("ip_src_addr"));
+ }
+ for (int i = 5; i < 10; ++i) {
+ Map<String, Object> source = results.get(i).getSource();
+ Assert.assertEquals(1, source.size());
+ Assert.assertNotNull(source.get("ip_src_addr"));
+ }
+ }
+
+ @Test
public void sort_by_guid() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(sortByGuidQuery, SearchRequest.class);
SearchResponse response = getIndexDao().search(request);
@@ -906,7 +923,7 @@ public abstract class SearchIntegrationTest {
for (int i = 0; i < 5; ++i) {
Map<String, Object> source = results.get(i).getSource();
Assert.assertEquals(1, source.size());
- Assert.assertEquals(source.get(Constants.GUID), "bro_" + (i + 1));
+ Assert.assertEquals(source.get("guid"), "bro_" + (i + 1));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/0e037eda/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
index 58e9db4..84f2222 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
@@ -34,7 +33,6 @@ import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
-import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.integration.InMemoryComponent;
import org.apache.metron.solr.dao.SolrDao;
import org.apache.metron.solr.integration.components.SolrComponent;
@@ -226,23 +224,6 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
}
@Test
- public void queries_fields() throws Exception {
- SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class);
- SearchResponse response = getIndexDao().search(request);
- Assert.assertEquals(10, response.getTotal());
-
- List<SearchResult> results = response.getResults();
- Assert.assertEquals(10, response.getResults().size());
-
- // validate the source fields contained in the search response
- for (int i = 0; i < 10; ++i) {
- Map<String, Object> source = results.get(i).getSource();
- Assert.assertNotNull(source);
- Assert.assertNotNull(source.get(Constants.Fields.SRC_ADDR.getName()));
- }
- }
-
- @Test
public void different_type_filter_query() throws Exception {
thrown.expect(InvalidSearchException.class);
SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFilterQuery, SearchRequest.class);
[40/51] [abbrv] metron git commit: METRON-1821 Align
prepare-release-candidate with documentation (JonZeolla via jonzeolla) closes
apache/metron#1236
Posted by ot...@apache.org.
METRON-1821 Align prepare-release-candidate with documentation (JonZeolla via jonzeolla) closes apache/metron#1236
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/64789660
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/64789660
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/64789660
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 647896604f938ca195bf25885651f9be3e897c71
Parents: 90c5e1d
Author: JonZeolla <ze...@gmail.com>
Authored: Mon Oct 15 09:42:44 2018 -0400
Committer: jonzeolla <jo...@apache.org>
Committed: Mon Oct 15 09:42:44 2018 -0400
----------------------------------------------------------------------
.../release-utils/prepare-release-candidate | 23 +++++++++-----------
1 file changed, 10 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/64789660/dev-utilities/release-utils/prepare-release-candidate
----------------------------------------------------------------------
diff --git a/dev-utilities/release-utils/prepare-release-candidate b/dev-utilities/release-utils/prepare-release-candidate
index bedee9b..635285c 100755
--- a/dev-utilities/release-utils/prepare-release-candidate
+++ b/dev-utilities/release-utils/prepare-release-candidate
@@ -33,9 +33,10 @@ BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${BRO_PLUGIN_REPO_N
DEV_REPO="https://dist.apache.org/repos/dist/dev/metron"
RELEASE_REPO=" https://dist.apache.org/repos/dist/release/metron"
-PLUGIN_GIT_REPO="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git"
+PLUGIN_GIT_REPO="https://git-wip-us.apache.org/repos/asf/${BRO_PLUGIN_REPO_NAME}.git"
RC_PREFIX=rc
+UPPER_RC_PREFIX="$(tr '[:lower:]' '[:upper:]' <<< ${RC_PREFIX})"
TAG_POSTFIX="-release"
CONFIG_FILE=~/.metron-prepare-release-candidate
@@ -223,33 +224,29 @@ if [ "${PRACTICE_RUN}" = true ]; then
printf "This is a practice run. Not running <git push --set-upstream origin %s_%s>\n" "${CAPITAL_REPO}" "$VERSION"
else
printf "Pushing branch %s_%s\n" "${CAPITAL_REPO}" "$VERSION"
- git push --set-upstream origin "${BRANCH_PREFIX}_${BRANCH_VERSION}"
+ git push --set-upstream origin "${CAPITAL_REPO}_${VERSION}"
fi
# Create directory for release artifacts
if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
- ART_DIR="$WORKDIR/dev/metron/${VERSION}-RC${RC_NUM}"
+ ART_DIR="$WORKDIR/dev/metron/${VERSION}-${UPPER_RC_PREFIX}${RC_NUM}"
else
# We're using a sub module, so put it in it's own directory.
- ART_DIR="$WORKDIR/dev/metron/${CHOSEN_REPO}/${VERSION}-RC${RC_NUM}"
+ ART_DIR="$WORKDIR/dev/metron/${CHOSEN_REPO}/${VERSION}-${UPPER_RC_PREFIX}${RC_NUM}"
fi
mkdir -p "$ART_DIR"
# Setup various parameters we need for the release artifacts
if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
- CORE_PREFIX="apache-metron-"
+ CORE_PREFIX="apache-metron_"
ARTIFACT_PREFIX="${CORE_PREFIX}"
TAG_VERSION="${CURRENT_VERSION}"
TAG="${CORE_PREFIX}${TAG_VERSION}${TAG_POSTFIX}"
elif [ "${CHOSEN_REPO}" = "${BRO_PLUGIN_REPO_NAME}" ]; then
- PLUGIN_PREFIX="apache-metron-bro-plugin-kafka_"
- ARTIFACT_PREFIX="${PLUGIN_PREFIX}"
+ BRO_PLUGIN_PREFIX="apache-metron-bro-plugin-kafka_"
+ ARTIFACT_PREFIX="${BRO_PLUGIN_PREFIX}"
TAG_VERSION="${CURRENT_VERSION}"
- TAG="${PLUGIN_PREFIX}${TAG_VERSION}${TAG_POSTFIX}"
- # Handle special tag case from prior release
- if [ "${TAG_VERSION}" = "0.1" ]; then
- TAG="0.1"
- fi
+ TAG="${BRO_PLUGIN_PREFIX}${TAG_VERSION}${TAG_POSTFIX}"
else
# If we ever add new modules, add them as needed.
printf "Unrecognized module: %s\n" "${CHOSEN_REPO}"
@@ -324,7 +321,7 @@ if [ "${PRACTICE_RUN}" = true ]; then
printf "<svn add %s>\n" ${COMMIT_DIR}
printf "<svn commit -m \"Adding artifacts for %s %s\">\n" "${CHOSEN_REPO}" "${COMMIT_DIR}"
else
- printf "Adding artifacts for Metron ${VERSION}-RC${RC_NUM} to dev SVN\n"
+ printf "Adding artifacts for Metron ${VERSION}-${UPPER_RC_PREFIX}${RC_NUM} to dev SVN\n"
# Metron goes in the root of the dir, submodules go in folder
if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then
cd "$WORKDIR/dev/metron/"
[07/51] [abbrv] metron git commit: METRON-1774 Allow user to
configure JAAS client in Ambari (nickwallen) closes apache/metron#1192
Posted by ot...@apache.org.
METRON-1774 Allow user to configure JAAS client in Ambari (nickwallen) closes apache/metron#1192
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b84c8740
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b84c8740
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b84c8740
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: b84c8740db5fbd05f3876aee3cb5532622a6b9dd
Parents: 91c410f
Author: nickwallen <ni...@nickallen.org>
Authored: Tue Sep 18 11:35:13 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Tue Sep 18 11:35:13 2018 -0400
----------------------------------------------------------------------
.../configuration/metron-client-jaas-conf.xml | 61 ++++++++++++++++++++
.../common-services/METRON/CURRENT/metainfo.xml | 1 +
.../CURRENT/package/scripts/metron_security.py | 4 +-
.../package/scripts/params/params_linux.py | 2 +
.../package/templates/client_jaas.conf.j2 | 44 --------------
5 files changed, 66 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/b84c8740/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-client-jaas-conf.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-client-jaas-conf.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-client-jaas-conf.xml
new file mode 100644
index 0000000..141a5eb
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-client-jaas-conf.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+ <property>
+ <name>content</name>
+ <display-name>metron_client_jaas template</display-name>
+ <description>Metron client JAAS configuration</description>
+ <value>
+StormClient {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=false
+ storeKey=false
+ useTicketCache=true
+ serviceName="nimbus"
+ principal="{{metron_principal_name}}";
+};
+Client {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{metron_keytab_path}}"
+ storeKey=true
+ useTicketCache=false
+ serviceName="zookeeper"
+ principal="{{metron_principal_name}}";
+};
+KafkaClient {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{metron_keytab_path}}"
+ storeKey=true
+ useTicketCache=false
+ serviceName="kafka"
+ principal="{{metron_principal_name}}";
+};
+ </value>
+ <value-attributes>
+ <type>content</type>
+ <show-property-name>false</show-property-name>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/b84c8740/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index f83d93b..644ba97 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -471,6 +471,7 @@
<config-type>kafka-broker</config-type>
<config-type>kafka-env</config-type>
<config-type>zeppelin-config</config-type>
+ <config-type>metron-client-jaas-conf</config-type>
</configuration-dependencies>
<restartRequiredAfterChange>true</restartRequiredAfterChange>
<quickLinksConfigurations>
http://git-wip-us.apache.org/repos/asf/metron/blob/b84c8740/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py
index 4f04daf..d9486e8 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_security.py
@@ -16,6 +16,7 @@ limitations under the License.
import os.path
from resource_management.core.source import Template
+from resource_management.core.source import InlineTemplate
from resource_management.core.resources.system import Directory, File
from resource_management.core import global_lock
from resource_management.core.logger import Logger
@@ -46,7 +47,7 @@ def storm_security_setup(params):
)
File(ambari_format('{client_jaas_path}'),
- content=Template('client_jaas.conf.j2'),
+ content=InlineTemplate(params.metron_client_jaas_conf_template),
owner=params.metron_user,
group=params.metron_group,
mode=0755
@@ -80,4 +81,3 @@ def kinit(kinit_path_local, keytab_path, principal_name, execute_user=None):
Execute(kinitcmd, user=execute_user)
finally:
kinit_lock.release()
-
http://git-wip-us.apache.org/repos/asf/metron/blob/b84c8740/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 9be09f1..0525c7f 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -259,6 +259,8 @@ if security_enabled:
kafka_principal_name = kafka_principal_raw.replace('_HOST', hostname_lowercase)
kafka_keytab_path = config['configurations']['kafka-env']['kafka_keytab']
+ metron_client_jaas_conf_template = config['configurations']['metron-client-jaas-conf']['content']
+
nimbus_seeds = config['configurations']['storm-site']['nimbus.seeds']
# Check wether Solr mpack is installed
if 'solr-config-env' in config['configurations']:
http://git-wip-us.apache.org/repos/asf/metron/blob/b84c8740/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/client_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/client_jaas.conf.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/client_jaas.conf.j2
deleted file mode 100644
index c0a047e..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/client_jaas.conf.j2
+++ /dev/null
@@ -1,44 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-StormClient {
- com.sun.security.auth.module.Krb5LoginModule required
- useKeyTab=false
- storeKey=false
- useTicketCache=true
- serviceName="nimbus"
- principal="{{metron_principal_name}}";
-};
-Client {
- com.sun.security.auth.module.Krb5LoginModule required
- useKeyTab=true
- keyTab="{{metron_keytab_path}}"
- storeKey=true
- useTicketCache=false
- serviceName="zookeeper"
- principal="{{metron_principal_name}}";
-};
-KafkaClient {
- com.sun.security.auth.module.Krb5LoginModule required
- useKeyTab=true
- keyTab="{{metron_keytab_path}}"
- storeKey=true
- useTicketCache=false
- serviceName="kafka"
- principal="{{metron_principal_name}}";
-};
[05/51] [abbrv] metron git commit: METRON-1777: Fix Elasticsearch
X-Pack sample pom in documentation (mmiklavc via mmiklavc) closes
apache/metron#1196
Posted by ot...@apache.org.
METRON-1777: Fix Elasticsearch X-Pack sample pom in documentation (mmiklavc via mmiklavc) closes apache/metron#1196
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8b2f297f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8b2f297f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8b2f297f
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 8b2f297fb58df1c2f01fa34a9d2ed5ccd183a7ce
Parents: 02a1280
Author: mmiklavc <mi...@gmail.com>
Authored: Tue Sep 18 08:55:45 2018 -0600
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Tue Sep 18 08:55:45 2018 -0600
----------------------------------------------------------------------
metron-deployment/Kerberos-manual-setup.md | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/8b2f297f/metron-deployment/Kerberos-manual-setup.md
----------------------------------------------------------------------
diff --git a/metron-deployment/Kerberos-manual-setup.md b/metron-deployment/Kerberos-manual-setup.md
index d34a62a..4bd4516 100644
--- a/metron-deployment/Kerberos-manual-setup.md
+++ b/metron-deployment/Kerberos-manual-setup.md
@@ -30,7 +30,7 @@ This document provides instructions for kerberizing Metron's Vagrant-based devel
* [Start Metron](#start-metron)
* [Push Data](#push-data)
* [More Information](#more-information)
-* [Elasticseach X-Pack](#x-pack)
+* [Elasticseach X-Pack](#X-Pack)
Setup
-----
@@ -568,7 +568,7 @@ X-Pack
Add the `es.client.settings` to global.json
```
- /usr/metron/0.6.0/config/zookeeper/global.json ->
+ $METRON_HOME/config/zookeeper/global.json ->
"es.client.settings" : {
"es.client.class" : "org.elasticsearch.xpack.client.PreBuiltXPackTransportClient",
@@ -650,10 +650,6 @@ X-Pack
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
- <exclusion> <!-- this is causing a weird build error if not excluded - Error creating shaded jar: null: IllegalArgumentException -->
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- </exclusion>
</exclusions>
</dependency>
</dependencies>
@@ -662,7 +658,7 @@ X-Pack
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>2.4.3</version>
+ <version>3.2.0</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
@@ -728,7 +724,8 @@ X-Pack
1. Once you've built the `elasticsearch-xpack-shaded-5.6.2.jar`, it needs to be made available to Storm when you submit the topology. Create a contrib directory for indexing and put the jar file in this directory.
```
- /usr/metron/0.6.0/indexing_contrib/elasticsearch-xpack-shaded-5.6.2.jar
+ mkdir $METRON_HOME/indexing_contrib
+ cp elasticsearch-xpack-shaded-5.6.2.jar $METRON_HOME/indexing_contrib/elasticsearch-xpack-shaded-5.6.2.jar
```
1. Now you can restart the Elasticsearch topology. Note, you should perform this step manually, as follows.
[04/51] [abbrv] metron git commit: Merge remote-tracking branch
'apache/master' into feature/METRON-1699-create-batch-profiler
Posted by ot...@apache.org.
Merge remote-tracking branch 'apache/master' into feature/METRON-1699-create-batch-profiler
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/cad2f408
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/cad2f408
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/cad2f408
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: cad2f4086a29f25a5ae2aeaf70d0933776df3681
Parents: fb623f6 02a1280
Author: Nick Allen <ni...@nickallen.org>
Authored: Tue Sep 18 10:53:41 2018 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Tue Sep 18 10:53:41 2018 -0400
----------------------------------------------------------------------
.gitignore | 8 +
metron-analytics/metron-maas-common/pom.xml | 2 +-
metron-analytics/metron-maas-service/pom.xml | 2 +-
metron-analytics/metron-profiler-client/pom.xml | 2 +-
metron-analytics/metron-profiler-common/pom.xml | 2 +-
.../profiler/DefaultMessageDistributor.java | 18 +-
.../src/test/resources/log4j.properties | 3 +
metron-analytics/metron-profiler-repl/pom.xml | 2 +-
metron-analytics/metron-profiler-spark/pom.xml | 2 +-
metron-analytics/metron-profiler-storm/pom.xml | 2 +-
.../profiler/storm/ProfileBuilderBolt.java | 71 +-
.../zookeeper/event-time-test/profiler.json | 19 +-
.../profiler/storm/ProfileBuilderBoltTest.java | 22 +
.../integration/ProfilerIntegrationTest.java | 351 +-
.../src/test/resources/log4j.properties | 10 +-
.../src/test/resources/telemetry.json | 100 +
metron-analytics/metron-statistics/pom.xml | 2 +-
metron-analytics/pom.xml | 2 +-
metron-contrib/metron-docker/pom.xml | 2 +-
metron-contrib/metron-performance/pom.xml | 2 +-
metron-contrib/pom.xml | 2 +-
metron-deployment/Kerberos-manual-setup.md | 4 +-
metron-deployment/amazon-ec2/conf/defaults.yml | 2 +-
.../ansible/playbooks/docker_probe_install.yml | 2 +-
.../roles/ambari_master/defaults/main.yml | 2 +-
.../centos6/ansible/inventory/group_vars/all | 2 +-
.../ubuntu14/ansible/inventory/group_vars/all | 2 +-
.../ambari/elasticsearch-mpack/pom.xml | 4 +-
.../packaging/ambari/metron-mpack/pom.xml | 4 +-
.../metron-mpack/src/main/resources/mpack.json | 2 +-
.../packaging/docker/deb-docker/pom.xml | 4 +-
.../docker/rpm-docker/SPECS/metron.spec | 6 +-
.../packaging/docker/rpm-docker/pom.xml | 4 +-
metron-deployment/pom.xml | 2 +-
metron-interface/metron-alerts/.gitignore | 49 +-
metron-interface/metron-alerts/.nvmrc | 1 +
metron-interface/metron-alerts/README.md | 16 +-
metron-interface/metron-alerts/angular-cli.json | 65 -
metron-interface/metron-alerts/angular.json | 153 +
.../metron-alerts/e2e/tsconfig.json | 17 +-
metron-interface/metron-alerts/karma.conf.js | 18 +-
.../metron-alerts/package-lock.json | 24240 ++++++++++++-----
metron-interface/metron-alerts/package.json | 67 +-
metron-interface/metron-alerts/pom.xml | 8 +-
.../metron-alerts/protractor.conf.js | 12 +-
.../scripts/prepend_license_header.sh | 2 +-
.../metron-alerts/scripts/start-dev.sh | 2 +-
.../scripts/start-server-for-e2e.sh | 2 +-
.../alert-details/alert-details.component.ts | 2 +-
.../alerts/alerts-list/alerts-list.component.ts | 6 +-
.../table-view/table-view.component.spec.ts | 21 +-
.../table-view/table-view.component.ts | 2 +-
.../alerts-list/tree-view/tree-group-data.ts | 2 +-
.../tree-view/tree-view.component.spec.ts | 6 +-
.../tree-view/tree-view.component.ts | 2 +-
.../configure-table.component.ts | 4 +-
.../alerts/meta-alerts/meta-alerts.component.ts | 2 +-
.../saved-searches.component.spec.ts | 9 +-
.../saved-searches/saved-searches.component.ts | 5 +-
.../metron-alerts/src/app/app-routing.module.ts | 2 +-
.../metron-alerts/src/app/app.component.spec.ts | 4 +-
.../metron-alerts/src/app/app.module.ts | 8 +-
.../default-headers.interceptor.ts | 34 +
.../src/app/login/login.component.spec.ts | 4 +-
.../metron-alerts/src/app/model/rest-error.ts | 4 +-
.../pcap/pcap-filters/pcap-filters.component.ts | 5 +-
.../pcap-panel/pcap-panel.component.spec.ts | 9 +-
.../app/pcap/pcap-panel/pcap-panel.component.ts | 4 +-
.../metron-alerts/src/app/pcap/pcap.module.ts | 14 +-
.../src/app/pcap/service/pcap.service.spec.ts | 236 +-
.../src/app/pcap/service/pcap.service.ts | 115 +-
.../src/app/service/alerts.service.ts | 17 +-
.../src/app/service/authentication.service.ts | 44 +-
.../src/app/service/cluster-metadata.service.ts | 9 +-
.../src/app/service/column-names.service.ts | 6 +-
.../src/app/service/configure-table.service.ts | 8 +-
.../src/app/service/data-source.ts | 7 +-
.../service/elasticsearch-localstorage-impl.ts | 36 +-
.../src/app/service/global-config.service.ts | 21 +-
.../src/app/service/meta-alert.service.ts | 43 +-
.../src/app/service/save-search.service.ts | 8 +-
.../src/app/service/search.service.ts | 54 +-
.../src/app/service/update.service.ts | 41 +-
.../metron-alerts/src/app/shared/auth-guard.ts | 2 +-
.../metron-alerts/src/app/shared/index.ts | 18 -
.../src/app/utils/elasticsearch-utils.ts | 4 +-
.../metron-alerts/src/app/utils/httpUtil.ts | 21 +-
.../metron-alerts/src/global-shim.ts | 18 +
metron-interface/metron-alerts/src/main.ts | 5 +-
metron-interface/metron-alerts/src/polyfills.ts | 2 +
.../metron-alerts/src/tsconfig.app.json | 3 +
.../metron-alerts/src/tsconfig.spec.json | 3 +-
metron-interface/metron-config/README.md | 5 +-
.../metron-config/package-lock.json | 2 +-
metron-interface/metron-config/package.json | 2 +-
metron-interface/metron-config/pom.xml | 2 +-
.../metron-config/scripts/package.json | 2 +-
metron-interface/metron-rest-client/pom.xml | 2 +-
metron-interface/metron-rest/pom.xml | 2 +-
.../src/main/resources/application.yml | 2 +-
metron-interface/pom.xml | 2 +-
metron-platform/Performance-tuning-guide.md | 4 +-
metron-platform/README.md | 2 +-
metron-platform/elasticsearch-shaded/pom.xml | 2 +-
metron-platform/metron-common/pom.xml | 2 +-
.../src/main/scripts/cluster_info.py | 2 +-
metron-platform/metron-data-management/pom.xml | 2 +-
metron-platform/metron-elasticsearch/pom.xml | 2 +-
metron-platform/metron-enrichment/pom.xml | 2 +-
metron-platform/metron-hbase-client/pom.xml | 2 +-
metron-platform/metron-hbase/pom.xml | 2 +-
metron-platform/metron-indexing/pom.xml | 2 +-
metron-platform/metron-integration-test/pom.xml | 2 +-
metron-platform/metron-job/pom.xml | 2 +-
metron-platform/metron-management/pom.xml | 2 +-
metron-platform/metron-parsers/pom.xml | 2 +-
metron-platform/metron-pcap-backend/pom.xml | 2 +-
.../src/main/scripts/pcap_zeppelin_run.sh | 2 +-
metron-platform/metron-pcap/pom.xml | 2 +-
metron-platform/metron-solr/pom.xml | 2 +-
.../metron-storm-kafka-override/pom.xml | 2 +-
metron-platform/metron-storm-kafka/pom.xml | 2 +-
metron-platform/metron-test-utilities/pom.xml | 2 +-
metron-platform/metron-writer/pom.xml | 2 +-
metron-platform/metron-zookeeper/pom.xml | 2 +-
metron-platform/pom.xml | 2 +-
metron-stellar/pom.xml | 2 +-
.../stellar-3rd-party-example/pom.xml | 2 +-
metron-stellar/stellar-common/README.md | 4 +-
metron-stellar/stellar-common/pom.xml | 2 +-
metron-stellar/stellar-zeppelin/README.md | 6 +-
metron-stellar/stellar-zeppelin/pom.xml | 2 +-
pom.xml | 2 +-
site-book/bin/generate-md.sh | 17 +
site-book/pom.xml | 2 +-
site/current-book/CONTRIBUTING.html | 8 +-
site/current-book/Upgrading.html | 8 +-
site/current-book/css/maven-base.css | 168 -
site/current-book/css/maven-theme.css | 161 -
site/current-book/css/print.css | 11 +-
.../images/aggregated_parser_chaining_flow.svg | 14 +
site/current-book/images/clustered.png | Bin 0 -> 218476 bytes
site/current-book/images/collapsed.gif | Bin 53 -> 0 bytes
site/current-book/images/drill_down.png | Bin 0 -> 246210 bytes
site/current-book/images/expanded.gif | Bin 52 -> 0 bytes
site/current-book/images/external.png | Bin 230 -> 0 bytes
site/current-book/images/find_alerts.png | Bin 0 -> 581508 bytes
site/current-book/images/icon_error_sml.gif | Bin 1010 -> 633 bytes
site/current-book/images/icon_info_sml.gif | Bin 606 -> 638 bytes
site/current-book/images/icon_success_sml.gif | Bin 990 -> 604 bytes
site/current-book/images/icon_warning_sml.gif | Bin 576 -> 625 bytes
.../images/message_routing_high_level.svg | 14 +
.../metron-job_state_statechart_diagram.svg | 14 +
site/current-book/images/newwindow.png | Bin 220 -> 0 bytes
.../images/performance_measurement.png | Bin 0 -> 5790 bytes
site/current-book/images/squid_search.png | Bin 0 -> 161855 bytes
.../images/unified_enrichment_arch.svg | 14 +
site/current-book/index.html | 8 +-
site/current-book/metron-analytics/index.html | 8 +-
.../metron-maas-service/index.html | 8 +-
.../metron-profiler-client/index.html | 8 +-
.../metron-analytics/metron-profiler/index.html | 53 +-
.../metron-statistics/HLLP.html | 8 +-
.../metron-statistics/index.html | 8 +-
.../metron-contrib/metron-docker/index.html | 8 +-
.../metron-performance/index.html | 10 +-
.../Kerberos-ambari-setup.html | 8 +-
.../Kerberos-manual-setup.html | 97 +-
.../metron-deployment/amazon-ec2/index.html | 8 +-
.../metron-deployment/ansible/index.html | 8 +-
.../metron-deployment/ansible/roles/index.html | 8 +-
.../ansible/roles/opentaxii/index.html | 8 +-
.../ansible/roles/pcap_replay/index.html | 8 +-
.../ansible/roles/sensor-stubs/index.html | 8 +-
.../ansible/roles/sensor-test-mode/index.html | 8 +-
.../development/centos6/index.html | 12 +-
.../development/fastcapa/index.html | 8 +-
.../metron-deployment/development/index.html | 8 +-
.../development/ubuntu14/index.html | 12 +-
site/current-book/metron-deployment/index.html | 13 +-
.../metron-deployment/other-examples/index.html | 8 +-
.../manual-install/Manual_Install_CentOS6.html | 8 +-
.../ambari/elasticsearch-mpack/index.html | 8 +-
.../packaging/ambari/index.html | 8 +-
.../packaging/ambari/metron-mpack/index.html | 8 +-
.../packaging/docker/ansible-docker/index.html | 8 +-
.../packaging/docker/deb-docker/index.html | 8 +-
.../packaging/docker/rpm-docker/index.html | 8 +-
.../packaging/packer-build/index.html | 8 +-
.../metron-interface/metron-alerts/index.html | 69 +-
.../metron-interface/metron-config/index.html | 13 +-
.../metron-interface/metron-rest/index.html | 204 +-
.../Performance-tuning-guide.html | 15 +-
site/current-book/metron-platform/index.html | 13 +-
.../metron-platform/metron-api/index.html | 161 -
.../metron-platform/metron-common/index.html | 43 +-
.../metron-data-management/index.html | 11 +-
.../metron-elasticsearch/index.html | 19 +-
.../metron-enrichment/Performance.html | 11 +-
.../metron-enrichment/index.html | 25 +-
.../metron-platform/metron-indexing/index.html | 65 +-
.../metron-platform/metron-job/index.html | 126 +
.../metron-management/index.html | 11 +-
.../metron-parsers/3rdPartyParser.html | 87 +-
.../metron-parsers/ParserChaining.html | 259 +
.../metron-platform/metron-parsers/index.html | 131 +-
.../metron-parsers/parser-testing.html | 12 +-
.../metron-pcap-backend/index.html | 15 +-
.../metron-platform/metron-solr/index.html | 300 +
.../metron-platform/metron-writer/index.html | 33 +-
.../metron-sensors/fastcapa/index.html | 8 +-
site/current-book/metron-sensors/index.html | 8 +-
.../metron-sensors/pycapa/index.html | 181 +-
.../stellar-3rd-party-example/index.html | 8 +-
.../stellar-common/3rdPartyStellar.html | 8 +-
.../metron-stellar/stellar-common/index.html | 27 +-
.../metron-stellar/stellar-zeppelin/index.html | 14 +-
.../use-cases/forensic_clustering/index.html | 245 +-
.../geographic_login_outliers/index.html | 14 +-
site/current-book/use-cases/index.html | 9 +-
.../use-cases/parser_chaining/index.html | 412 +
.../use-cases/typosquat_detection/index.html | 25 +-
site/documentation/index.md | 15 +-
223 files changed, 20489 insertions(+), 9225 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-repl/pom.xml
----------------------------------------------------------------------
diff --cc metron-analytics/metron-profiler-repl/pom.xml
index 2447b40,0000000..0876f70
mode 100644,000000..100644
--- a/metron-analytics/metron-profiler-repl/pom.xml
+++ b/metron-analytics/metron-profiler-repl/pom.xml
@@@ -1,150 -1,0 +1,150 @@@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>metron-analytics</artifactId>
+ <groupId>org.apache.metron</groupId>
- <version>0.5.1</version>
++ <version>0.6.0</version>
+ </parent>
+ <artifactId>metron-profiler-repl</artifactId>
+ <url>https://metron.apache.org/</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-profiler-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-profiler-client</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <!-- allows profiles to use the Stellar stats functions -->
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-statistics</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>kryo</artifactId>
+ <groupId>com.esotericsoftware</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>${global_log4j_core_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${global_log4j_core_version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${global_shade_version}</version>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resources>
+ <resource>.yaml</resource>
+ <resource>LICENSE.txt</resource>
+ <resource>ASL2.0</resource>
+ <resource>NOTICE.txt</resource>
+ </resources>
+ </transformer>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass></mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-spark/pom.xml
----------------------------------------------------------------------
diff --cc metron-analytics/metron-profiler-spark/pom.xml
index 387dce4,0000000..587b38c
mode 100644,000000..100644
--- a/metron-analytics/metron-profiler-spark/pom.xml
+++ b/metron-analytics/metron-profiler-spark/pom.xml
@@@ -1,216 -1,0 +1,216 @@@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-analytics</artifactId>
- <version>0.5.1</version>
++ <version>0.6.0</version>
+ </parent>
+ <artifactId>metron-profiler-spark</artifactId>
+ <url>https://metron.apache.org/</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.11</artifactId>
+ <version>${global_spark_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_2.11</artifactId>
+ <version>${global_spark_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr-runtime</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-profiler-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-profiler-client</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-hbase</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-hbase</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-hbase</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${global_hbase_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <!-- allows profiles to use the Stellar stats functions -->
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-statistics</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>kryo</artifactId>
+ <groupId>com.esotericsoftware</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>${global_log4j_core_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${global_log4j_core_version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${global_shade_version}</version>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>com.tdunning</pattern>
+ <shadedPattern>org.apache.metron.tdunning</shadedPattern>
+ </relocation>
+ </relocations>
+ <artifactSet>
+ <excludes>
+ <exclude>storm:storm-core:*</exclude>
+ <exclude>storm:storm-lib:*</exclude>
+ <exclude>org.slf4j.impl*</exclude>
+ <exclude>org.slf4j:slf4j-log4j*</exclude>
+ </excludes>
+ </artifactSet>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resources>
+ <resource>.yaml</resource>
+ <resource>LICENSE.txt</resource>
+ <resource>ASL2.0</resource>
+ <resource>NOTICE.txt</resource>
+ </resources>
+ </transformer>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass></mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-storm/pom.xml
----------------------------------------------------------------------
diff --cc metron-analytics/metron-profiler-storm/pom.xml
index 22c6255,0000000..dd80467
mode 100644,000000..100644
--- a/metron-analytics/metron-profiler-storm/pom.xml
+++ b/metron-analytics/metron-profiler-storm/pom.xml
@@@ -1,407 -1,0 +1,407 @@@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-analytics</artifactId>
- <version>0.5.1</version>
++ <version>0.6.0</version>
+ </parent>
+ <artifactId>metron-profiler-storm</artifactId>
+ <url>https://metron.apache.org/</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <version>${global_hadoop_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${global_hadoop_version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>commons-httpclient</artifactId>
+ <groupId>commons-httpclient</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>asm</groupId>
+ <artifactId>asm</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${global_hadoop_version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-writer</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-storm-kafka</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-statistics</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>kryo</artifactId>
+ <groupId>com.esotericsoftware</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-profiler-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-profiler-client</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-hbase</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-hbase</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.esotericsoftware</groupId>
+ <artifactId>kryo-shaded</artifactId>
+ <version>${global_kryo_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${global_hadoop_version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>commons-httpclient</artifactId>
+ <groupId>commons-httpclient</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${global_hbase_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${global_storm_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>kryo</artifactId>
+ <groupId>com.esotericsoftware</groupId>
+ </exclusion>
+ </exclusions>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>${global_log4j_core_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${global_log4j_core_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ <version>${global_kafka_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${global_kafka_version}</version>
+ <classifier>test</classifier>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${global_mockito_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.tempus-fugit</groupId>
+ <artifactId>tempus-fugit</artifactId>
+ <version>1.2-20140129.191141-5</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-integration-test</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-test-utilities</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${global_shade_version}</version>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>org.apache.metron.guava.metron-profiler</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>org.apache.metron.jackson</shadedPattern>
+ </relocation>
+ </relocations>
+ <artifactSet>
+ <excludes>
+ <exclude>storm:storm-core:*</exclude>
+ <exclude>storm:storm-lib:*</exclude>
+ <exclude>org.slf4j.impl*</exclude>
+ <exclude>org.slf4j:slf4j-log4j*</exclude>
+ </excludes>
+ </artifactSet>
+ <transformers>
+ <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resources>
+ <resource>.yaml</resource>
+ <resource>LICENSE.txt</resource>
+ <resource>ASL2.0</resource>
+ <resource>NOTICE.txt</resource>
+ </resources>
+ </transformer>
+ <!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
+ <!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
+ <addHeader>false</addHeader>
+ <projectName>${project.name}</projectName>
+ </transformer-->
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass></mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.atteo.classindex</groupId>
+ <artifactId>classindex-transformer</artifactId>
+ <version>${global_classindex_version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java
----------------------------------------------------------------------
diff --cc metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java
index 205159e,0000000..a4cd4f8
mode 100644,000000..100644
--- a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java
+++ b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileBuilderBolt.java
@@@ -1,509 -1,0 +1,552 @@@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.storm;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.ConfigurationType;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.configuration.profiler.ProfileConfig;
+import org.apache.metron.common.configuration.profiler.ProfilerConfigurations;
+import org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater;
+import org.apache.metron.common.zookeeper.configurations.ProfilerUpdater;
+import org.apache.metron.common.zookeeper.configurations.Reloadable;
+import org.apache.metron.profiler.DefaultMessageDistributor;
+import org.apache.metron.profiler.MessageDistributor;
+import org.apache.metron.profiler.MessageRoute;
+import org.apache.metron.profiler.ProfileMeasurement;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.zookeeper.SimpleEventListener;
+import org.apache.metron.zookeeper.ZKCache;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.topology.base.BaseWindowedBolt;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.windowing.TupleWindow;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.List;
++import java.util.LongSummaryStatistics;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
++import java.util.stream.Collectors;
+
+import static java.lang.String.format;
+import static org.apache.metron.profiler.storm.ProfileSplitterBolt.ENTITY_TUPLE_FIELD;
+import static org.apache.metron.profiler.storm.ProfileSplitterBolt.MESSAGE_TUPLE_FIELD;
+import static org.apache.metron.profiler.storm.ProfileSplitterBolt.PROFILE_TUPLE_FIELD;
+import static org.apache.metron.profiler.storm.ProfileSplitterBolt.TIMESTAMP_TUPLE_FIELD;
+
+/**
+ * A Storm bolt that is responsible for building a profile.
+ *
+ * <p>This bolt maintains the state required to build a Profile. When the window
+ * period expires, the data is summarized as a {@link ProfileMeasurement}, all state is
+ * flushed, and the {@link ProfileMeasurement} is emitted.
++ *
++ * <p>There are two mechanisms that will cause a profile to flush. As new messages arrive,
++ * time is advanced. The splitter bolt attaches a timestamp to each message (which can be
++ * either event or system time.) This advances time and leads to profile measurements
++ * being flushed. Alternatively, if no messages arrive to advance time, then the "time-to-live"
++ * mechanism will flush a profile after no messages have been received for some period of time.
+ */
+public class ProfileBuilderBolt extends BaseWindowedBolt implements Reloadable {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private OutputCollector collector;
+
+ /**
+ * The URL to connect to Zookeeper.
+ */
+ private String zookeeperUrl;
+
+ /**
+ * The Zookeeper client connection.
+ */
+ protected CuratorFramework zookeeperClient;
+
+ /**
+ * The Zookeeper cache.
+ */
+ protected ZKCache zookeeperCache;
+
+ /**
+ * Manages configuration for the Profiler.
+ */
+ private ProfilerConfigurations configurations;
+
+ /**
+ * The duration of each profile period in milliseconds.
+ */
+ private long periodDurationMillis;
+
+ /**
+ * The duration of Storm's event window.
+ */
+ private long windowDurationMillis;
+
+ /**
+ * If a message has not been applied to a Profile in this number of milliseconds,
+ * the Profile will be forgotten and its resources will be cleaned up.
+ *
+ * <p>WARNING: The TTL must be at least greater than the period duration.
+ */
+ private long profileTimeToLiveMillis;
+
+ /**
+ * The maximum number of {@link MessageRoute} routes that will be maintained by
+ * this bolt. After this value is exceeded, lesser used routes will be evicted
+ * from the internal cache.
+ */
+ private long maxNumberOfRoutes;
+
+ /**
+ * Distributes messages to the profile builders.
+ *
+ * <p>Since expired profiles are flushed on a separate thread, all access to this
+ * {@code MessageDistributor} needs to be protected.
+ */
+ private MessageDistributor messageDistributor;
+
+ /**
+ * Parses JSON messages.
+ */
+ private transient JSONParser parser;
+
+ /**
+ * Responsible for emitting {@link ProfileMeasurement} values.
+ *
+ * <p>The {@link ProfileMeasurement} values generated by a profile can be written to
+ * multiple endpoints like HBase or Kafka. Each endpoint is handled by a separate
+ * {@link ProfileMeasurementEmitter}.
+ */
+ private List<ProfileMeasurementEmitter> emitters;
+
+ /**
+ * Signals when it is time to flush the active profiles.
+ */
+ private FlushSignal activeFlushSignal;
+
+ /**
+ * An executor that flushes expired profiles at a regular interval on a separate
+ * thread.
+ *
+ * <p>Flushing expired profiles ensures that any profiles that stop receiving messages
+ * for an extended period of time will continue to be flushed.
+ *
+ * <p>This introduces concurrency issues as the bolt is no longer single threaded. Due
+ * to this, all access to the {@code MessageDistributor} needs to be protected.
+ */
+ private transient ScheduledExecutorService flushExpiredExecutor;
+
+ public ProfileBuilderBolt() {
+ this.emitters = new ArrayList<>();
+ }
+
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ super.prepare(stormConf, context, collector);
+
+ if(periodDurationMillis <= 0) {
+ throw new IllegalArgumentException("expect 'profiler.period.duration' >= 0");
+ }
+ if(profileTimeToLiveMillis <= 0) {
+ throw new IllegalArgumentException("expect 'profiler.ttl' >= 0");
+ }
+ if(profileTimeToLiveMillis < periodDurationMillis) {
+ throw new IllegalArgumentException("expect 'profiler.ttl' >= 'profiler.period.duration'");
+ }
+ if(maxNumberOfRoutes <= 0) {
+ throw new IllegalArgumentException("expect 'profiler.max.routes.per.bolt' > 0");
+ }
+ if(windowDurationMillis <= 0) {
+ throw new IllegalArgumentException("expect 'profiler.window.duration' > 0");
+ }
+ if(windowDurationMillis > periodDurationMillis) {
+ throw new IllegalArgumentException("expect 'profiler.period.duration' >= 'profiler.window.duration'");
+ }
+ if(periodDurationMillis % windowDurationMillis != 0) {
+ throw new IllegalArgumentException("expect 'profiler.period.duration' % 'profiler.window.duration' == 0");
+ }
+
+ this.collector = collector;
+ this.parser = new JSONParser();
+ this.messageDistributor = new DefaultMessageDistributor(periodDurationMillis, profileTimeToLiveMillis, maxNumberOfRoutes);
+ this.configurations = new ProfilerConfigurations();
+ this.activeFlushSignal = new FixedFrequencyFlushSignal(periodDurationMillis);
+ setupZookeeper();
+ startFlushingExpiredProfiles();
+ }
+
+ @Override
+ public void cleanup() {
+ try {
+ zookeeperCache.close();
+ zookeeperClient.close();
+ flushExpiredExecutor.shutdown();
+
+ } catch(Throwable e) {
+ LOG.error("Exception when cleaning up", e);
+ }
+ }
+
+ /**
+ * Setup connectivity to Zookeeper which provides the necessary configuration for the bolt.
+ */
+ private void setupZookeeper() {
+ try {
+ if (zookeeperClient == null) {
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ zookeeperClient = CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy);
+ }
+ zookeeperClient.start();
+
+ // this is temporary to ensure that any validation passes. the individual bolt
+ // will reinitialize stellar to dynamically pull from zookeeper.
+ ConfigurationsUtils.setupStellarStatically(zookeeperClient);
+ if (zookeeperCache == null) {
+ ConfigurationsUpdater<ProfilerConfigurations> updater = createUpdater();
+ SimpleEventListener listener = new SimpleEventListener.Builder()
+ .with( updater::update, TreeCacheEvent.Type.NODE_ADDED, TreeCacheEvent.Type.NODE_UPDATED)
+ .with( updater::delete, TreeCacheEvent.Type.NODE_REMOVED)
+ .build();
+ zookeeperCache = new ZKCache.Builder()
+ .withClient(zookeeperClient)
+ .withListener(listener)
+ .withRoot(Constants.ZOOKEEPER_TOPOLOGY_ROOT)
+ .build();
+ updater.forceUpdate(zookeeperClient);
+ zookeeperCache.start();
+ }
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected ConfigurationsUpdater<ProfilerConfigurations> createUpdater() {
+ return new ProfilerUpdater(this, this::getConfigurations);
+ }
+
+ public ProfilerConfigurations getConfigurations() {
+ return configurations;
+ }
+
+ @Override
+ public void reloadCallback(String name, ConfigurationType type) {
+ // nothing to do
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+
+ if(emitters.size() == 0) {
+ throw new IllegalStateException("At least one destination handler must be defined.");
+ }
+
+ // allow each emitter to define its own stream
+ emitters.forEach(emitter -> emitter.declareOutputFields(declarer));
+ }
+
+ private Context getStellarContext() {
+
+ Map<String, Object> global = getConfigurations().getGlobalConfig();
+ return new Context.Builder()
+ .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> zookeeperClient)
+ .with(Context.Capabilities.GLOBAL_CONFIG, () -> global)
+ .with(Context.Capabilities.STELLAR_CONFIG, () -> global)
+ .build();
+ }
+
++ /**
++ * Logs information about the {@link TupleWindow}.
++ *
++ * @param window The tuple window.
++ */
++ private void log(TupleWindow window) {
++ // summarize the newly received tuples
++ LongSummaryStatistics received = window.get()
++ .stream()
++ .map(tuple -> getField(TIMESTAMP_TUPLE_FIELD, tuple, Long.class))
++ .collect(Collectors.summarizingLong(Long::longValue));
++
++ LOG.debug("Tuple(s) received; count={}, min={}, max={}, range={} ms",
++ received.getCount(),
++ received.getMin(),
++ received.getMax(),
++ received.getMax() - received.getMin());
++
++ if (window.getExpired().size() > 0) {
++ // summarize the expired tuples
++ LongSummaryStatistics expired = window.getExpired()
++ .stream()
++ .map(tuple -> getField(TIMESTAMP_TUPLE_FIELD, tuple, Long.class))
++ .collect(Collectors.summarizingLong(Long::longValue));
++
++ LOG.debug("Tuple(s) expired; count={}, min={}, max={}, range={} ms, lag={} ms",
++ expired.getCount(),
++ expired.getMin(),
++ expired.getMax(),
++ expired.getMax() - expired.getMin(),
++ received.getMin() - expired.getMin());
++ }
++ }
++
+ @Override
+ public void execute(TupleWindow window) {
-
- LOG.debug("Tuple window contains {} tuple(s), {} expired, {} new",
- CollectionUtils.size(window.get()),
- CollectionUtils.size(window.getExpired()),
- CollectionUtils.size(window.getNew()));
++ if(LOG.isDebugEnabled()) {
++ log(window);
++ }
+
+ try {
-
+ // handle each tuple in the window
+ for(Tuple tuple : window.get()) {
+ handleMessage(tuple);
+ }
+
+ // time to flush active profiles?
+ if(activeFlushSignal.isTimeToFlush()) {
+ flushActive();
+ }
+
+ } catch (Throwable e) {
-
+ LOG.error("Unexpected error", e);
+ collector.reportError(e);
+ }
+ }
+
+ /**
+ * Flush all active profiles.
+ */
+ protected void flushActive() {
+ activeFlushSignal.reset();
+
+ // flush the active profiles
+ List<ProfileMeasurement> measurements;
+ synchronized(messageDistributor) {
+ measurements = messageDistributor.flush();
+ emitMeasurements(measurements);
+ }
+
+ LOG.debug("Flushed active profiles and found {} measurement(s).", measurements.size());
-
+ }
+
+ /**
+ * Flushes all expired profiles.
+ *
+ * <p>If a profile has not received a message for an extended period of time then it is
+ * marked as expired. Periodically we need to flush these expired profiles to ensure
+ * that their state is not lost.
+ */
+ protected void flushExpired() {
++ List<ProfileMeasurement> measurements = null;
++ try {
++ // flush the expired profiles
++ synchronized (messageDistributor) {
++ measurements = messageDistributor.flushExpired();
++ emitMeasurements(measurements);
++ }
+
- // flush the expired profiles
- List<ProfileMeasurement> measurements;
- synchronized (messageDistributor) {
- measurements = messageDistributor.flushExpired();
- emitMeasurements(measurements);
++ } catch(Throwable t) {
++ // need to catch the exception, otherwise subsequent executions would be suppressed.
++ // see java.util.concurrent.ScheduledExecutorService#scheduleAtFixedRate
++ LOG.error("Failed to flush expired profiles", t);
+ }
+
- LOG.debug("Flushed expired profiles and found {} measurement(s).", measurements.size());
++ LOG.debug("Flushed expired profiles and found {} measurement(s).", CollectionUtils.size(measurements));
+ }
+
+ /**
+ * Handles the processing of a single tuple.
+ *
+ * @param input The tuple containing a telemetry message.
+ */
+ private void handleMessage(Tuple input) {
+
+ // crack open the tuple
+ JSONObject message = getField(MESSAGE_TUPLE_FIELD, input, JSONObject.class);
+ ProfileConfig definition = getField(PROFILE_TUPLE_FIELD, input, ProfileConfig.class);
+ String entity = getField(ENTITY_TUPLE_FIELD, input, String.class);
+ Long timestamp = getField(TIMESTAMP_TUPLE_FIELD, input, Long.class);
+
+ // keep track of time
+ activeFlushSignal.update(timestamp);
+
+ // distribute the message
+ MessageRoute route = new MessageRoute(definition, entity, message, timestamp);
+ synchronized (messageDistributor) {
+ messageDistributor.distribute(route, getStellarContext());
+ }
+
+ LOG.debug("Message distributed: profile={}, entity={}, timestamp={}", definition.getProfile(), entity, timestamp);
+ }
+
+ /**
+ * Handles the {@code ProfileMeasurement}s that are created when a profile is flushed.
+ *
+ * @param measurements The measurements to handle.
+ */
+ private void emitMeasurements(List<ProfileMeasurement> measurements) {
+
+ // flush each profile
+ for(ProfileMeasurement measurement: measurements) {
+
+ // allow each 'emitter' to emit the measurement
+ for (ProfileMeasurementEmitter emitter : emitters) {
+ emitter.emit(measurement, collector);
+
+ LOG.debug("Measurement emitted; stream={}, profile={}, entity={}, value={}, start={}, end={}, duration={}, period={}",
+ emitter.getStreamId(),
+ measurement.getProfileName(),
+ measurement.getEntity(),
+ measurement.getProfileValue(),
+ measurement.getPeriod().getStartTimeMillis(),
+ measurement.getPeriod().getEndTimeMillis(),
+ measurement.getPeriod().getDurationMillis(),
+ measurement.getPeriod().getPeriod());
+ }
+ }
+
+ LOG.debug("Emitted {} measurement(s).", measurements.size());
+ }
+
+ /**
+ * Retrieves an expected field from a Tuple. If the field is missing an exception is thrown to
+ * indicate a fatal error.
+ * @param fieldName The name of the field.
+ * @param tuple The tuple from which to retrieve the field.
+ * @param clazz The type of the field value.
+ * @param <T> The type of the field value.
+ */
+ private <T> T getField(String fieldName, Tuple tuple, Class<T> clazz) {
+
+ T value = ConversionUtils.convert(tuple.getValueByField(fieldName), clazz);
+ if(value == null) {
+ throw new IllegalStateException(format("Invalid tuple: missing or invalid field '%s'", fieldName));
+ }
+
+ return value;
+ }
+
+ /**
+ * Creates a separate thread that regularly flushes expired profiles.
+ */
+ private void startFlushingExpiredProfiles() {
+
+ long initialDelay = profileTimeToLiveMillis;
+ long period = profileTimeToLiveMillis;
+ flushExpiredExecutor = Executors.newSingleThreadScheduledExecutor();
+ flushExpiredExecutor.scheduleAtFixedRate(() -> flushExpired(), initialDelay, period, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public BaseWindowedBolt withTumblingWindow(BaseWindowedBolt.Duration duration) {
+
+ // need to capture the window duration to validate it along with other profiler settings
+ this.windowDurationMillis = duration.value;
+ return super.withTumblingWindow(duration);
+ }
+
+ public long getPeriodDurationMillis() {
+ return periodDurationMillis;
+ }
+
+ public ProfileBuilderBolt withPeriodDurationMillis(long periodDurationMillis) {
+ this.periodDurationMillis = periodDurationMillis;
+ return this;
+ }
+
+ public ProfileBuilderBolt withPeriodDuration(int duration, TimeUnit units) {
+ return withPeriodDurationMillis(units.toMillis(duration));
+ }
+
+ public ProfileBuilderBolt withProfileTimeToLiveMillis(long timeToLiveMillis) {
+ this.profileTimeToLiveMillis = timeToLiveMillis;
+ return this;
+ }
+
+ public long getWindowDurationMillis() {
+ return windowDurationMillis;
+ }
+
+ public ProfileBuilderBolt withProfileTimeToLive(int duration, TimeUnit units) {
+ return withProfileTimeToLiveMillis(units.toMillis(duration));
+ }
+
+ public ProfileBuilderBolt withEmitter(ProfileMeasurementEmitter emitter) {
+ this.emitters.add(emitter);
+ return this;
+ }
+
+ public MessageDistributor getMessageDistributor() {
+ return messageDistributor;
+ }
+
+ public ProfileBuilderBolt withZookeeperUrl(String zookeeperUrl) {
+ this.zookeeperUrl = zookeeperUrl;
+ return this;
+ }
+
+ public ProfileBuilderBolt withZookeeperClient(CuratorFramework zookeeperClient) {
+ this.zookeeperClient = zookeeperClient;
+ return this;
+ }
+
+ public ProfileBuilderBolt withZookeeperCache(ZKCache zookeeperCache) {
+ this.zookeeperCache = zookeeperCache;
+ return this;
+ }
+
+ public ProfileBuilderBolt withProfilerConfigurations(ProfilerConfigurations configurations) {
+ this.configurations = configurations;
+ return this;
+ }
+
+ public ProfileBuilderBolt withMaxNumberOfRoutes(long maxNumberOfRoutes) {
+ this.maxNumberOfRoutes = maxNumberOfRoutes;
+ return this;
+ }
+
+ public ProfileBuilderBolt withFlushSignal(FlushSignal flushSignal) {
+ this.activeFlushSignal = flushSignal;
+ return this;
+ }
+
+ public ProfileBuilderBolt withMessageDistributor(MessageDistributor messageDistributor) {
+ this.messageDistributor = messageDistributor;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json
----------------------------------------------------------------------
diff --cc metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json
index 9d727a3,0000000..534b7c6
mode 100644,000000..100644
--- a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json
+++ b/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/event-time-test/profiler.json
@@@ -1,12 -1,0 +1,19 @@@
+{
++ "timestampField": "timestamp",
+ "profiles": [
+ {
- "profile": "event-time-test",
++ "profile": "count-by-ip",
+ "foreach": "ip_src_addr",
- "init": { "counter": "0" },
- "update": { "counter": "counter + 1" },
- "result": "counter"
++ "init": { "count": 0 },
++ "update": { "count" : "count + 1" },
++ "result": "count"
++ },
++ {
++ "profile": "total-count",
++ "foreach": "'total'",
++ "init": { "count": 0 },
++ "update": { "count": "count + 1" },
++ "result": "count"
+ }
- ],
- "timestampField": "timestamp"
++ ]
+}
[43/51] [abbrv] metron git commit: METRON-1681 Decouple the
ParserBolt from the Parse execution logic (merrimanr) closes
apache/metron#1213
Posted by ot...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
new file mode 100644
index 0000000..5f05b24
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
@@ -0,0 +1,390 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.error.MetronError;
+import org.apache.metron.common.message.metadata.RawMessage;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.common.utils.ReflectionUtils;
+import org.apache.metron.parsers.filters.Filters;
+import org.apache.metron.parsers.filters.StellarFilter;
+import org.apache.metron.parsers.interfaces.MessageFilter;
+import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.apache.metron.parsers.topology.ParserComponent;
+import org.apache.metron.parsers.ParserRunnerImpl.ProcessResult;
+import org.apache.metron.stellar.dsl.Context;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ParserRunnerImpl.class, ReflectionUtils.class, Filters.class})
+public class ParserRunnerImplTest {
+
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ /**
+ {
+ "fieldValidations" : [
+ {
+ "input" : [ "ip_src_addr", "ip_dst_addr"],
+ "validation" : "IP"
+ }
+ ]
+ }
+ */
+ @Multiline
+ private String globalConfigString;
+
+ /**
+ {
+ "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
+ "filterClassName":"org.apache.metron.parsers.filters.StellarFilter",
+ "sensorTopic":"bro",
+ "parserConfig": {
+ "field": "value"
+ },
+ "fieldTransformations" : [
+ {
+ "input" : "field1",
+ "transformation" : "REMOVE"
+ }
+ ]
+ }
+ */
+ @Multiline
+ private String broConfigString;
+
+ /**
+ {
+ "parserClassName":"org.apache.metron.parsers.snort.BasicSnortParser",
+ "sensorTopic":"snort",
+ "parserConfig": {}
+ }
+ */
+ @Multiline
+ private String snortConfigString;
+
+ private ParserConfigurations parserConfigurations;
+ private MessageParser<JSONObject> broParser;
+ private MessageParser<JSONObject> snortParser;
+ private MessageFilter<JSONObject> stellarFilter;
+ private ParserRunnerImpl parserRunner;
+
+
+ @Before
+ public void setup() throws IOException {
+ parserConfigurations = new ParserConfigurations();
+ SensorParserConfig broConfig = SensorParserConfig.fromBytes(broConfigString.getBytes());
+ SensorParserConfig snortConfig = SensorParserConfig.fromBytes(snortConfigString.getBytes());
+ parserConfigurations.updateSensorParserConfig("bro", broConfig);
+ parserConfigurations.updateSensorParserConfig("snort", snortConfig);
+ parserConfigurations.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfigString, JSONUtils.MAP_SUPPLIER));
+ parserRunner = new ParserRunnerImpl(new HashSet<>(Arrays.asList("bro", "snort")));
+ broParser = mock(MessageParser.class);
+ snortParser = mock(MessageParser.class);
+ stellarFilter = mock(StellarFilter.class);
+ mockStatic(ReflectionUtils.class);
+ mockStatic(Filters.class);
+
+ when(ReflectionUtils.createInstance("org.apache.metron.parsers.bro.BasicBroParser")).thenReturn(broParser);
+ when(ReflectionUtils.createInstance("org.apache.metron.parsers.snort.BasicSnortParser")).thenReturn(snortParser);
+ when(Filters.get("org.apache.metron.parsers.filters.StellarFilter", broConfig.getParserConfig()))
+ .thenReturn(stellarFilter);
+
+ }
+
+ @Test
+ public void shouldThrowExceptionOnEmptyParserSupplier() {
+ exception.expect(IllegalStateException.class);
+ exception.expectMessage("A parser config supplier must be set before initializing the ParserRunner.");
+
+ parserRunner.init(null, null);
+ }
+
+ @Test
+ public void shouldThrowExceptionOnEmptyStellarContext() {
+ exception.expect(IllegalStateException.class);
+ exception.expectMessage("A stellar context must be set before initializing the ParserRunner.");
+
+ parserRunner.init(() -> parserConfigurations, null);
+ }
+
+ @Test
+ public void initShouldThrowExceptionOnMissingSensorParserConfig() {
+ exception.expect(IllegalStateException.class);
+ exception.expectMessage("Could not initialize parsers. Cannot find configuration for sensor test.");
+
+ parserRunner = new ParserRunnerImpl(new HashSet<String>() {{
+ add("test");
+ }});
+
+ parserRunner.init(() -> parserConfigurations, mock(Context.class));
+ }
+
+ @Test
+ public void executeShouldThrowExceptionOnMissingSensorParserConfig() {
+ exception.expect(IllegalStateException.class);
+ exception.expectMessage("Could not execute parser. Cannot find configuration for sensor test.");
+
+ parserRunner = new ParserRunnerImpl(new HashSet<String>() {{
+ add("test");
+ }});
+
+ parserRunner.execute("test", mock(RawMessage.class), parserConfigurations);
+ }
+
+ @Test
+ public void shouldInit() throws Exception {
+ Context stellarContext = mock(Context.class);
+ Map<String, Object> broParserConfig = parserConfigurations.getSensorParserConfig("bro").getParserConfig();
+ Map<String, Object> snortParserConfig = parserConfigurations.getSensorParserConfig("snort").getParserConfig();
+
+ parserRunner.init(() -> parserConfigurations, stellarContext);
+
+ {
+ // Verify Stellar context
+ Assert.assertEquals(stellarContext, parserRunner.getStellarContext());
+ }
+
+ Map<String, ParserComponent> sensorToParserComponentMap = parserRunner.getSensorToParserComponentMap();
+ {
+ // Verify Bro parser initialization
+ Assert.assertEquals(2, sensorToParserComponentMap.size());
+ ParserComponent broComponent = sensorToParserComponentMap.get("bro");
+ Assert.assertEquals(broParser, broComponent.getMessageParser());
+ Assert.assertEquals(stellarFilter, broComponent.getFilter());
+ verify(broParser, times(1)).init();
+ verify(broParser, times(1)).configure(broParserConfig);
+ verifyNoMoreInteractions(broParser);
+ verifyNoMoreInteractions(stellarFilter);
+ }
+ {
+ // Verify Snort parser initialization
+ ParserComponent snortComponent = sensorToParserComponentMap.get("snort");
+ Assert.assertEquals(snortParser, snortComponent.getMessageParser());
+ Assert.assertNull(snortComponent.getFilter());
+ verify(snortParser, times(1)).init();
+ verify(snortParser, times(1)).configure(snortParserConfig);
+ verifyNoMoreInteractions(snortParser);
+ }
+ }
+
+ @Test
+ public void shouldExecute() {
+ parserRunner = spy(parserRunner);
+ RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>());
+ JSONObject parsedMessage1 = new JSONObject();
+ parsedMessage1.put("field", "parsedMessage1");
+ JSONObject parsedMessage2 = new JSONObject();
+ parsedMessage2.put("field", "parsedMessage2");
+ Object rawMessage1 = new RawMessage("raw_message1".getBytes(), new HashMap<>());
+ Object rawMessage2 = new RawMessage("raw_message2".getBytes(), new HashMap<>());
+ Throwable throwable1 = mock(Throwable.class);
+ Throwable throwable2 = mock(Throwable.class);
+ MessageParserResult<JSONObject> messageParserResult = new DefaultMessageParserResult<>(Arrays.asList(parsedMessage1, parsedMessage2),
+ new HashMap<Object, Throwable>(){{
+ put(rawMessage1, throwable1);
+ put(rawMessage2, throwable2);
+ }});
+ JSONObject processedMessage = new JSONObject();
+ processedMessage.put("field", "processedMessage1");
+ MetronError processedError = new MetronError().withMessage("processedError");
+ ProcessResult processedMessageResult = mock(ProcessResult.class);
+ ProcessResult processedErrorResult = mock(ProcessResult.class);
+
+ when(broParser.parseOptionalResult(rawMessage.getMessage())).thenReturn(Optional.of(messageParserResult));
+ when(processedMessageResult.getMessage()).thenReturn(processedMessage);
+ when(processedErrorResult.isError()).thenReturn(true);
+ when(processedErrorResult.getError()).thenReturn(processedError);
+ doReturn(Optional.of(processedMessageResult)).when(parserRunner)
+ .processMessage("bro", parsedMessage1, rawMessage, broParser, parserConfigurations);
+ doReturn(Optional.of(processedErrorResult)).when(parserRunner)
+ .processMessage("bro", parsedMessage2, rawMessage, broParser, parserConfigurations);
+
+ MetronError expectedParseError1 = new MetronError()
+ .withErrorType(Constants.ErrorType.PARSER_ERROR)
+ .withThrowable(throwable1)
+ .withSensorType(Collections.singleton("bro"))
+ .addRawMessage(rawMessage1);
+ MetronError expectedParseError2 = new MetronError()
+ .withErrorType(Constants.ErrorType.PARSER_ERROR)
+ .withThrowable(throwable2)
+ .withSensorType(Collections.singleton("bro"))
+ .addRawMessage(rawMessage2);
+
+ parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{
+ put("bro", new ParserComponent(broParser, stellarFilter));
+ }});
+ ParserRunnerResults<JSONObject> parserRunnerResults = parserRunner.execute("bro", rawMessage, parserConfigurations);
+
+ Assert.assertEquals(1, parserRunnerResults.getMessages().size());
+ Assert.assertTrue(parserRunnerResults.getMessages().contains(processedMessage));
+ Assert.assertEquals(3, parserRunnerResults.getErrors().size());
+ Assert.assertTrue(parserRunnerResults.getErrors().contains(processedError));
+ Assert.assertTrue(parserRunnerResults.getErrors().contains(expectedParseError1));
+ Assert.assertTrue(parserRunnerResults.getErrors().contains(expectedParseError2));
+ }
+
+ @Test
+ public void shouldExecuteWithMasterThrowable() {
+ parserRunner = spy(parserRunner);
+ RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>());
+ Throwable masterThrowable = mock(Throwable.class);
+ MessageParserResult<JSONObject> messageParserResult = new DefaultMessageParserResult<>(masterThrowable);
+
+
+ when(broParser.parseOptionalResult(rawMessage.getMessage())).thenReturn(Optional.of(messageParserResult));
+
+ parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{
+ put("bro", new ParserComponent(broParser, stellarFilter));
+ }});
+ ParserRunnerResults<JSONObject> parserRunnerResults = parserRunner.execute("bro", rawMessage, parserConfigurations);
+
+ verify(parserRunner, times(0))
+ .processMessage(any(), any(), any(), any(), any());
+
+ MetronError expectedError = new MetronError()
+ .withErrorType(Constants.ErrorType.PARSER_ERROR)
+ .withThrowable(masterThrowable)
+ .withSensorType(Collections.singleton("bro"))
+ .addRawMessage(rawMessage.getMessage());
+ Assert.assertEquals(1, parserRunnerResults.getErrors().size());
+ Assert.assertTrue(parserRunnerResults.getErrors().contains(expectedError));
+ }
+
+ @Test
+ public void shouldPopulateMessagesOnProcessMessage() {
+ JSONObject inputMessage = new JSONObject();
+ inputMessage.put("guid", "guid");
+ inputMessage.put("ip_src_addr", "192.168.1.1");
+ inputMessage.put("ip_dst_addr", "192.168.1.2");
+ inputMessage.put("field1", "value");
+ RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>());
+
+ JSONObject expectedOutput = new JSONObject();
+ expectedOutput.put("guid", "guid");
+ expectedOutput.put("source.type", "bro");
+ expectedOutput.put("ip_src_addr", "192.168.1.1");
+ expectedOutput.put("ip_dst_addr", "192.168.1.2");
+
+ when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true);
+ when(broParser.validate(expectedOutput)).thenReturn(true);
+
+ parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{
+ put("bro", new ParserComponent(broParser, stellarFilter));
+ }});
+
+ Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations);
+
+ Assert.assertTrue(processResult.isPresent());
+ Assert.assertFalse(processResult.get().isError());
+ Assert.assertEquals(expectedOutput, processResult.get().getMessage());
+ }
+
+ @Test
+ public void shouldReturnMetronErrorOnInvalidMessage() {
+ JSONObject inputMessage = new JSONObject();
+ inputMessage.put("guid", "guid");
+ RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>());
+
+ JSONObject expectedOutput = new JSONObject();
+ expectedOutput.put("guid", "guid");
+ expectedOutput.put("source.type", "bro");
+ MetronError expectedMetronError = new MetronError()
+ .withErrorType(Constants.ErrorType.PARSER_INVALID)
+ .withSensorType(Collections.singleton("bro"))
+ .addRawMessage(inputMessage);
+
+ when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true);
+ when(broParser.validate(expectedOutput)).thenReturn(false);
+
+ parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{
+ put("bro", new ParserComponent(broParser, stellarFilter));
+ }});
+
+ Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations);
+
+ Assert.assertTrue(processResult.isPresent());
+ Assert.assertTrue(processResult.get().isError());
+ Assert.assertEquals(expectedMetronError, processResult.get().getError());
+ }
+
+ @Test
+ public void shouldReturnMetronErrorOnFailedFieldValidator() {
+ JSONObject inputMessage = new JSONObject();
+ inputMessage.put("guid", "guid");
+ inputMessage.put("ip_src_addr", "test");
+ inputMessage.put("ip_dst_addr", "test");
+ RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>());
+
+ JSONObject expectedOutput = new JSONObject();
+ expectedOutput.put("guid", "guid");
+ expectedOutput.put("ip_src_addr", "test");
+ expectedOutput.put("ip_dst_addr", "test");
+ expectedOutput.put("source.type", "bro");
+ MetronError expectedMetronError = new MetronError()
+ .withErrorType(Constants.ErrorType.PARSER_INVALID)
+ .withSensorType(Collections.singleton("bro"))
+ .addRawMessage(inputMessage)
+ .withErrorFields(new HashSet<>(Arrays.asList("ip_src_addr", "ip_dst_addr")));
+
+ when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true);
+ when(broParser.validate(expectedOutput)).thenReturn(true);
+
+ parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{
+ put("bro", new ParserComponent(broParser, stellarFilter));
+ }});
+
+ Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations);
+
+ Assert.assertTrue(processResult.isPresent());
+ Assert.assertTrue(processResult.get().isError());
+ Assert.assertEquals(expectedMetronError, processResult.get().getError());
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
index e5e7180..9f58d1c 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
@@ -17,719 +17,428 @@
*/
package org.apache.metron.parsers.bolt;
-import com.google.common.collect.ImmutableList;
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.common.configuration.IndexingConfigurations;
import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.configuration.SensorParserConfig;
-import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.error.MetronError;
-import org.apache.metron.common.writer.BulkMessageWriter;
-import org.apache.metron.common.writer.BulkWriterResponse;
-import org.apache.metron.common.writer.MessageWriter;
-import org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater;
-import org.apache.metron.parsers.DefaultMessageParserResult;
-import org.apache.metron.parsers.interfaces.MessageFilter;
-import org.apache.metron.parsers.interfaces.MessageParserResult;
-import org.apache.metron.parsers.interfaces.MultilineMessageParser;
-import org.apache.metron.parsers.topology.ParserComponents;
+import org.apache.metron.common.message.MessageGetStrategy;
+import org.apache.metron.common.message.metadata.RawMessage;
+import org.apache.metron.parsers.DefaultParserRunnerResults;
+import org.apache.metron.parsers.ParserRunnerResults;
+import org.apache.metron.parsers.ParserRunnerImpl;
import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration;
import org.apache.metron.test.bolt.BaseBoltTest;
import org.apache.metron.test.error.MetronErrorJSONMatcher;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
+import org.apache.storm.Config;
import org.apache.storm.tuple.Tuple;
import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.mockito.Mock;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
+import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
+import java.util.function.Supplier;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.argThat;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class ParserBoltTest extends BaseBoltTest {
- @Mock
- private MultilineMessageParser<JSONObject> parser;
-
- @Mock
- private MessageWriter<JSONObject> writer;
-
- @Mock
- private BulkMessageWriter<JSONObject> batchWriter;
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
@Mock
- private MessageFilter<JSONObject> filter;
+ private Tuple t1;
@Mock
- private Tuple t1;
+ private ParserRunnerImpl parserRunner;
@Mock
- private Tuple t2;
+ private WriterHandler writerHandler;
@Mock
- private Tuple t3;
+ private WriterHandler writerHandlerHandleAck;
@Mock
- private Tuple t4;
+ private MessageGetStrategy messageGetStrategy;
@Mock
- private Tuple t5;
+ private Context stellarContext;
- private static class RecordingWriter implements BulkMessageWriter<JSONObject> {
- List<JSONObject> records = new ArrayList<>();
+ private class MockParserRunner extends ParserRunnerImpl {
- @Override
- public void init(Map stormConf, TopologyContext topologyContext, WriterConfiguration config) throws Exception {
+ private boolean isInvalid = false;
+ private RawMessage rawMessage;
+ private JSONObject message;
+ public MockParserRunner(HashSet<String> sensorTypes) {
+ super(sensorTypes);
}
@Override
- public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception {
- records.addAll(messages);
- BulkWriterResponse ret = new BulkWriterResponse();
- ret.addAllSuccesses(tuples);
- return ret;
+ public ParserRunnerResults<JSONObject> execute(String sensorType, RawMessage rawMessage, ParserConfigurations parserConfigurations) {
+ DefaultParserRunnerResults parserRunnerResults = new DefaultParserRunnerResults();
+ this.rawMessage = rawMessage;
+ if (!isInvalid) {
+ parserRunnerResults.addMessage(message);
+ } else {
+ MetronError error = new MetronError()
+ .withErrorType(Constants.ErrorType.PARSER_INVALID)
+ .withSensorType(Collections.singleton(sensorType))
+ .addRawMessage(message);
+ parserRunnerResults.addError(error);
+ }
+ return parserRunnerResults;
}
- @Override
- public String getName() {
- return "recording";
+ protected void setInvalid(boolean isInvalid) {
+ this.isInvalid = isInvalid;
}
- @Override
- public void close() throws Exception {
-
+ protected void setMessage(JSONObject message) {
+ this.message = message;
}
- public List<JSONObject> getRecords() {
- return records;
+ protected RawMessage getRawMessage() {
+ return rawMessage;
}
}
- private static ConfigurationsUpdater<ParserConfigurations> createUpdater() {
- return createUpdater(Optional.empty());
- }
- private static ConfigurationsUpdater<ParserConfigurations> createUpdater(Optional<Integer> batchSize) {
- return new ConfigurationsUpdater<ParserConfigurations>(null, null) {
- @Override
- public void update(CuratorFramework client, String path, byte[] data) throws IOException { }
+ @Before
+ public void setup() {
+ when(writerHandler.handleAck()).thenReturn(false);
+ when(writerHandlerHandleAck.handleAck()).thenReturn(true);
- @Override
- public void delete(CuratorFramework client, String path, byte[] data) throws IOException { }
-
- @Override
- public ConfigurationType getType() {
- return ConfigurationType.PARSER;
- }
-
- @Override
- public void update(String name, byte[] data) throws IOException { }
-
- @Override
- public void delete(String name) { }
-
- @Override
- public Class<ParserConfigurations> getConfigurationClass() {
- return ParserConfigurations.class;
- }
-
- @Override
- public void forceUpdate(CuratorFramework client) { }
-
- @Override
- public ParserConfigurations defaultConfigurations() {
- return new ParserConfigurations() {
- @Override
- public SensorParserConfig getSensorParserConfig(String sensorType) {
- return new SensorParserConfig() {
- @Override
- public Map<String, Object> getParserConfig() {
- return new HashMap<String, Object>() {{
- if(batchSize.isPresent()) {
- put(IndexingConfigurations.BATCH_SIZE_CONF, batchSize.get());
- }
- }};
- }
- };
- }
- };
- }
- };
}
-
@Test
- public void testEmpty() throws Exception {
- String sensorType = "yaf";
- Map<String, ParserComponents> parserMap = Collections.singletonMap(
- sensorType,
- new ParserComponents(
- parser,
- null,
- new WriterHandler(writer)
- )
- );
- ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserMap) {
- @Override
- protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
- return ParserBoltTest.createUpdater();
- }
- };
-
- parserBolt.setCuratorFramework(client);
- parserBolt.setZKCache(cache);
- parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
- verify(parser, times(1)).init();
- verify(writer, times(1)).init();
- byte[] sampleBinary = "some binary message".getBytes();
-
- when(tuple.getBinary(0)).thenReturn(sampleBinary);
- when(parser.parseOptionalResult(sampleBinary)).thenReturn(null);
- parserBolt.execute(tuple);
- verify(parser, times(0)).validate(any());
- verify(writer, times(0)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), any());
- verify(outputCollector, times(1)).ack(tuple);
-
- MetronError error = new MetronError()
- .withErrorType(Constants.ErrorType.PARSER_ERROR)
- .withThrowable(new NullPointerException())
- .withSensorType(Collections.singleton(sensorType))
- .addRawMessage(sampleBinary);
- verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
+ public void shouldThrowExceptionOnDifferentHandleAck() {
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage("All writers must match when calling handleAck()");
+
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserRunner, new HashMap<String, WriterHandler>() {{
+ put("yaf", writerHandler);
+ put("bro", writerHandlerHandleAck);
+ }});
}
@Test
- public void testInvalid() throws Exception {
- String sensorType = "yaf";
- Map<String, ParserComponents> parserMap = Collections.singletonMap(
- sensorType,
- new ParserComponents(
- parser,
- null,
- new WriterHandler(writer)
- )
- );
- ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserMap) {
- @Override
- protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
- return ParserBoltTest.createUpdater();
- }
- };
+ public void withBatchTimeoutDivisorShouldSetBatchTimeoutDivisor() {
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserRunner, new HashMap<String, WriterHandler>() {{
+ put("yaf", writerHandler);
+ }}).withBatchTimeoutDivisor(5);
- buildGlobalConfig(parserBolt);
+ Assert.assertEquals(5, parserBolt.getBatchTimeoutDivisor());
+ }
- parserBolt.setCuratorFramework(client);
- parserBolt.setZKCache(cache);
- parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
- byte[] sampleBinary = "some binary message".getBytes();
-
- when(tuple.getBinary(0)).thenReturn(sampleBinary);
- JSONObject parsedMessage = new JSONObject();
- parsedMessage.put("field", "invalidValue");
- parsedMessage.put("guid", "this-is-unique-identifier-for-tuple");
- List<JSONObject> messageList = new ArrayList<>();
- messageList.add(parsedMessage);
- when(parser.parseOptionalResult(sampleBinary)).thenReturn(Optional.of(new DefaultMessageParserResult<>(messageList)));
- when(parser.validate(parsedMessage)).thenReturn(true);
- parserBolt.execute(tuple);
+ @Test
+ public void shouldThrowExceptionOnInvalidBatchTimeoutDivisor() {
+ exception.expect(IllegalArgumentException.class);
+ exception.expectMessage("batchTimeoutDivisor must be positive. Value provided was -1");
- MetronError error = new MetronError()
- .withErrorType(Constants.ErrorType.PARSER_INVALID)
- .withSensorType(Collections.singleton(sensorType))
- .withErrorFields(new HashSet<String>() {{ add("field"); }})
- .addRawMessage(new JSONObject(){{
- put("field", "invalidValue");
- put("source.type", "yaf");
- put("guid", "this-is-unique-identifier-for-tuple");
- }});
- verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserRunner, new HashMap<String, WriterHandler>() {{
+ put("yaf", writerHandler);
+ }}).withBatchTimeoutDivisor(-1);
}
@Test
- public void test() throws Exception {
- String sensorType = "yaf";
- Map<String, ParserComponents> parserMap = Collections.singletonMap(
- sensorType,
- new ParserComponents(
- parser,
- null,
- new WriterHandler(writer)
- )
- );
- ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserMap) {
+ public void shouldGetComponentConfiguration() {
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserRunner, new HashMap<String, WriterHandler>() {{
+ put("yaf", writerHandler);
+ }}) {
+
@Override
- protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
- return ParserBoltTest.createUpdater();
+ public ParserConfigurations getConfigurations() {
+ ParserConfigurations configurations = new ParserConfigurations();
+ SensorParserConfig sensorParserConfig = new SensorParserConfig();
+ sensorParserConfig.setParserConfig(new HashMap<String, Object>() {{
+ put(IndexingConfigurations.BATCH_SIZE_CONF, 10);
+ }});
+ configurations.updateSensorParserConfig("yaf", sensorParserConfig);
+ return configurations;
}
};
- parserBolt.setCuratorFramework(client);
- parserBolt.setZKCache(cache);
- parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
- verify(parser, times(1)).init();
- verify(writer, times(1)).init();
- byte[] sampleBinary = "some binary message".getBytes();
- JSONParser jsonParser = new JSONParser();
- final JSONObject sampleMessage1 = (JSONObject) jsonParser.parse("{ \"field1\":\"value1\", \"guid\": \"this-is-unique-identifier-for-tuple\" }");
- final JSONObject sampleMessage2 = (JSONObject) jsonParser.parse("{ \"field2\":\"value2\", \"guid\": \"this-is-unique-identifier-for-tuple\" }");
- List<JSONObject> messages = new ArrayList<JSONObject>() {{
- add(sampleMessage1);
- add(sampleMessage2);
- }};
- final JSONObject finalMessage1 = (JSONObject) jsonParser.parse("{ \"field1\":\"value1\", \"source.type\":\"" + sensorType + "\", \"guid\": \"this-is-unique-identifier-for-tuple\" }");
- final JSONObject finalMessage2 = (JSONObject) jsonParser.parse("{ \"field2\":\"value2\", \"source.type\":\"" + sensorType + "\", \"guid\": \"this-is-unique-identifier-for-tuple\" }");
- when(tuple.getBinary(0)).thenReturn(sampleBinary);
- when(parser.parseOptionalResult(sampleBinary)).thenReturn(Optional.of(new DefaultMessageParserResult<>(messages)));
- when(parser.validate(eq(messages.get(0)))).thenReturn(true);
- when(parser.validate(eq(messages.get(1)))).thenReturn(false);
- parserBolt.execute(tuple);
- verify(writer, times(1)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage1));
- verify(outputCollector, times(1)).ack(tuple);
- when(parser.validate(eq(messages.get(0)))).thenReturn(true);
- when(parser.validate(eq(messages.get(1)))).thenReturn(true);
- when(filter.emitTuple(eq(messages.get(0)), any())).thenReturn(false);
- when(filter.emitTuple(eq(messages.get(1)), any())).thenReturn(true);
- parserBolt.execute(tuple);
- verify(writer, times(1)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage2));
- verify(outputCollector, times(2)).ack(tuple);
- doThrow(new Exception()).when(writer).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage2));
- parserBolt.execute(tuple);
- verify(outputCollector, times(1)).reportError(any(Throwable.class));
- }
- /**
- {
- "filterClassName" : "STELLAR"
- ,"parserConfig" : {
- "filter.query" : "exists(field1)"
- }
- }
- */
- @Multiline
- public static String sensorParserConfig;
-
- /**
- * Tests to ensure that a message that is unfiltered results in one write and an ack.
- * @throws Exception
- */
- @Test
- public void testFilterSuccess() throws Exception {
- String sensorType = "yaf";
- Map<String, ParserComponents> parserMap = Collections.singletonMap(
- sensorType,
- new ParserComponents(
- parser,
- null,
- new WriterHandler(batchWriter)
- )
- );
- ParserBolt parserBolt = buildParserBolt(parserMap, sensorParserConfig);
-
- parserBolt.setCuratorFramework(client);
- parserBolt.setZKCache(cache);
- parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
- verify(parser, times(1)).init();
- verify(batchWriter, times(1)).init(any(), any(), any());
- BulkWriterResponse successResponse = mock(BulkWriterResponse.class);
- when(successResponse.getSuccesses()).thenReturn(ImmutableList.of(t1));
- when(batchWriter.write(any(), any(), any(), any())).thenReturn(successResponse);
- when(parser.validate(any())).thenReturn(true);
- when(parser.parseOptionalResult(any())).thenReturn(Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject(new HashMap<String, Object>() {{
- put("field1", "blah");
- }})))));
- parserBolt.execute(t1);
- verify(batchWriter, times(1)).write(any(), any(), any(), any());
- verify(outputCollector, times(1)).ack(t1);
+ Map<String, Object> componentConfiguration = parserBolt.getComponentConfiguration();
+ Assert.assertEquals(1, componentConfiguration.size());
+ Assert.assertEquals( 14, componentConfiguration.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS));
}
-
- /**
- * Tests to ensure that a message filtered out results in no writes, but an ack.
- * @throws Exception
- */
@Test
- public void testFilterFailure() throws Exception {
- String sensorType = "yaf";
- Map<String, ParserComponents> parserMap = Collections.singletonMap(
- sensorType,
- new ParserComponents(
- parser,
- null,
- new WriterHandler(batchWriter)
- )
- );
- ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserMap) {
+ public void shouldPrepare() {
+ Map stormConf = mock(Map.class);
+ SensorParserConfig yafConfig = mock(SensorParserConfig.class);
+ when(yafConfig.getSensorTopic()).thenReturn("yafTopic");
+ when(yafConfig.getParserConfig()).thenReturn(new HashMap<String, Object>() {{
+ put(IndexingConfigurations.BATCH_SIZE_CONF, 10);
+ }});
+ ParserConfigurations parserConfigurations = mock(ParserConfigurations.class);
+
+ ParserBolt parserBolt = spy(new ParserBolt("zookeeperUrl", parserRunner, new HashMap<String, WriterHandler>() {{
+ put("yaf", writerHandler);
+ }}) {
+
@Override
protected SensorParserConfig getSensorParserConfig(String sensorType) {
- try {
- return SensorParserConfig.fromBytes(Bytes.toBytes(sensorParserConfig));
- } catch (IOException e) {
- throw new RuntimeException(e);
+ if ("yaf".equals(sensorType)) {
+ return yafConfig;
}
+ return null;
}
@Override
- protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
- return ParserBoltTest.createUpdater();
+ public ParserConfigurations getConfigurations() {
+ return parserConfigurations;
}
- };
+ });
+ doReturn(stellarContext).when(parserBolt).initializeStellar();
parserBolt.setCuratorFramework(client);
parserBolt.setZKCache(cache);
- parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
- verify(parser, times(1)).init();
- verify(batchWriter, times(1)).init(any(), any(), any());
- when(parser.validate(any())).thenReturn(true);
- when(parser.parseOptionalResult(any())).thenReturn(Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject(new HashMap<String, Object>() {{
- put("field2", "blah");
- }})))));
- parserBolt.execute(t1);
- verify(batchWriter, times(0)).write(any(), any(), any(), any());
- verify(outputCollector, times(1)).ack(t1);
- }
- /**
- {
- "sensorTopic":"dummy"
- ,"parserConfig": {
- "batchSize" : 1
- }
- ,"fieldTransformations" : [
- {
- "transformation" : "STELLAR"
- ,"output" : "timestamp"
- ,"config" : {
- "timestamp" : "TO_EPOCH_TIMESTAMP(timestampstr, 'yyyy-MM-dd HH:mm:ss', 'UTC')"
- }
- }
- ]
- }
- */
- @Multiline
- public static String csvWithFieldTransformations;
- @Test
- public void testFieldTransformationPriorToValidation() {
- String sensorType = "dummy";
- RecordingWriter recordingWriter = new RecordingWriter();
- //create a parser which acts like a basic parser but returns no timestamp field.
- MultilineMessageParser<JSONObject> dummyParser = new MultilineMessageParser<JSONObject>() {
- @Override
- public void configure(Map<String, Object> config) {
- }
+ parserBolt.prepare(stormConf, topologyContext, outputCollector);
- @Override
- public void init() {
- }
+ verify(parserRunner, times(1)).init(any(Supplier.class), eq(stellarContext));
+ verify(yafConfig, times(1)).init();
+ Map<String, String> topicToSensorMap = parserBolt.getTopicToSensorMap();
+ Assert.assertEquals(1, topicToSensorMap.size());
+ Assert.assertEquals("yaf", topicToSensorMap.get("yafTopic"));
+ verify(writerHandler).init(stormConf, topologyContext, outputCollector, parserConfigurations);
+ verify(writerHandler).setDefaultBatchTimeout(14);
+ }
- @Override
- public boolean validate(JSONObject message) {
- Object timestampObject = message.get(Constants.Fields.TIMESTAMP.getName());
- if (timestampObject instanceof Long) {
- Long timestamp = (Long) timestampObject;
- return timestamp > 0;
- }
- return false;
- }
+ @Test
+ public void shouldThrowExceptionOnMissingConfig() {
+ exception.expect(IllegalStateException.class);
+ exception.expectMessage("Unable to retrieve a parser config for yaf");
- @Override
- @SuppressWarnings("unchecked")
- public Optional<MessageParserResult<JSONObject>> parseOptionalResult(byte[] rawMessage) {
- return Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject() {{
- put("data", "foo");
- put("timestampstr", "2016-01-05 17:02:30");
- put("original_string", "blah");
- }})));
- }
- };
+ Map stormConf = mock(Map.class);
- Map<String, ParserComponents> parserMap = Collections.singletonMap(
- sensorType,
- new ParserComponents(
- dummyParser,
- null,
- new WriterHandler(recordingWriter)
- )
- );
- ParserBolt parserBolt = buildParserBolt(parserMap, csvWithFieldTransformations);
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserRunner, new HashMap<String, WriterHandler>() {{
+ put("yaf", writerHandler);
+ }});
parserBolt.setCuratorFramework(client);
parserBolt.setZKCache(cache);
- parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
- when(t1.getBinary(0)).thenReturn(new byte[] {});
- parserBolt.execute(t1);
- Assert.assertEquals(1, recordingWriter.getRecords().size());
- long expected = 1452013350000L;
- Assert.assertEquals(expected, recordingWriter.getRecords().get(0).get("timestamp"));
+
+ parserBolt.prepare(stormConf, topologyContext, outputCollector);
}
+
@Test
- public void testDefaultBatchSize() throws Exception {
- String sensorType = "yaf";
- Map<String, ParserComponents> parserMap = Collections.singletonMap(
- sensorType,
- new ParserComponents(
- parser,
- filter,
- new WriterHandler(batchWriter)
- )
- );
- ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserMap) {
+ public void executeShouldHandleTickTuple() throws Exception {
+ when(t1.getSourceComponent()).thenReturn("__system");
+ when(t1.getSourceStreamId()).thenReturn("__tick");
+ ParserConfigurations parserConfigurations = mock(ParserConfigurations.class);
+
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserRunner, new HashMap<String, WriterHandler>() {{
+ put("yaf", writerHandler);
+ }}) {
+
@Override
- protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
- // this uses default batch size
- return ParserBoltTest.createUpdater();
+ public ParserConfigurations getConfigurations() {
+ return parserConfigurations;
}
};
- parserBolt.setCuratorFramework(client);
- parserBolt.setZKCache(cache);
- parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
- verify(parser, times(1)).init();
- verify(batchWriter, times(1)).init(any(), any(), any());
- when(parser.validate(any())).thenReturn(true);
- when(parser.parseOptionalResult(any())).thenReturn(Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject()))));
- when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
- BulkWriterResponse response = new BulkWriterResponse();
- Tuple[] uniqueTuples = new Tuple[ParserConfigurations.DEFAULT_KAFKA_BATCH_SIZE];
- for (int i=0; i < uniqueTuples.length; i++) {
- uniqueTuples[i] = mock(Tuple.class);
- response.addSuccess(uniqueTuples[i]);
- }
- when(batchWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(new HashSet<>(Arrays.asList(uniqueTuples))), any())).thenReturn(response);
- for (Tuple tuple : uniqueTuples) {
- parserBolt.execute(tuple);
- }
- for (Tuple uniqueTuple : uniqueTuples) {
- verify(outputCollector, times(1)).ack(uniqueTuple);
- }
+ parserBolt.setMessageGetStrategy(messageGetStrategy);
+ parserBolt.setOutputCollector(outputCollector);
+
+ parserBolt.execute(t1);
+
+ verify(writerHandler, times(1)).flush(parserConfigurations, messageGetStrategy);
+ verify(outputCollector, times(1)).ack(t1);
}
@Test
- public void testLessRecordsThanDefaultBatchSize() throws Exception {
- String sensorType = "yaf";
- Map<String, ParserComponents> parserMap = Collections.singletonMap(
- sensorType,
- new ParserComponents(
- parser,
- filter,
- new WriterHandler(batchWriter)
- )
- );
- ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserMap) {
+ public void shouldExecuteOnSuccess() throws Exception {
+ when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
+ when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
+ MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {{ add("yaf"); }});
+ ParserConfigurations parserConfigurations = new ParserConfigurations();
+ parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
+
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {{
+ put("yaf", writerHandler);
+ }}) {
+
@Override
- protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
- // this uses default batch size
- return ParserBoltTest.createUpdater();
+ public ParserConfigurations getConfigurations() {
+ return parserConfigurations;
}
};
- parserBolt.setCuratorFramework(client);
- parserBolt.setZKCache(cache);
- parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
- verify(parser, times(1)).init();
- verify(batchWriter, times(1)).init(any(), any(), any());
- when(parser.validate(any())).thenReturn(true);
- when(parser.parseOptionalResult(any())).thenReturn(Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject()))));
- when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
- int oneLessThanDefaultBatchSize = ParserConfigurations.DEFAULT_KAFKA_BATCH_SIZE - 1;
- BulkWriterResponse response = new BulkWriterResponse();
- Tuple[] uniqueTuples = new Tuple[oneLessThanDefaultBatchSize];
- for (int i=0; i < uniqueTuples.length; i++) {
- uniqueTuples[i] = mock(Tuple.class);
- response.addSuccess(uniqueTuples[i]);
+ parserBolt.setMessageGetStrategy(messageGetStrategy);
+ parserBolt.setOutputCollector(outputCollector);
+ parserBolt.setTopicToSensorMap(new HashMap<String, String>() {{
+ put("yafTopic", "yaf");
+ }});
+ JSONObject message = new JSONObject();
+ message.put("field", "value");
+ mockParserRunner.setMessage(message);
+ RawMessage expectedRawMessage = new RawMessage("originalMessage".getBytes(StandardCharsets.UTF_8), new HashMap<>());
+
+ {
+ // Verify the correct message is written and ack is handled
+ parserBolt.execute(t1);
+
+ Assert.assertEquals(expectedRawMessage, mockParserRunner.getRawMessage());
+ verify(writerHandler, times(1)).write("yaf", t1, message, parserConfigurations, messageGetStrategy);
+ verify(outputCollector, times(1)).ack(t1);
}
- for (Tuple tuple : uniqueTuples) {
- parserBolt.execute(tuple);
+ {
+ // Verify the tuple is not acked when the writer is set to handle ack
+ reset(outputCollector);
+ parserBolt.setSensorToWriterMap(new HashMap<String, WriterHandler>() {{
+ put("yaf", writerHandlerHandleAck);
+ }});
+
+ parserBolt.execute(t1);
+
+ verify(writerHandlerHandleAck, times(1)).write("yaf", t1, message, parserConfigurations, messageGetStrategy);
+ verify(outputCollector, times(0)).ack(t1);
}
- // should have no acking yet - batch size not fulfilled
- verify(outputCollector, never()).ack(any(Tuple.class));
- response.addSuccess(t1); // used to achieve count in final verify
- Iterable<Tuple> tuples = new HashSet(Arrays.asList(uniqueTuples)) {{
- add(t1);
- }};
- when(batchWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(tuples), any())).thenReturn(response);
- // meet batch size requirement and now it should ack
- parserBolt.execute(t1);
- verify(outputCollector, times(ParserConfigurations.DEFAULT_KAFKA_BATCH_SIZE)).ack(any(Tuple.class));
}
@Test
- public void testBatchOfOne() throws Exception {
- String sensorType = "yaf";
- Map<String, ParserComponents> parserMap = Collections.singletonMap(
- sensorType,
- new ParserComponents(
- parser,
- filter,
- new WriterHandler(batchWriter)
- )
- );
- ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserMap) {
+ public void shouldExecuteOnError() throws Exception {
+ when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
+ when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
+ MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {{
+ add("yaf");
+ }});
+ mockParserRunner.setInvalid(true);
+ ParserConfigurations parserConfigurations = new ParserConfigurations();
+ parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
+
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {{
+ put("yaf", writerHandler);
+ }}) {
+
@Override
- protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
- return ParserBoltTest.createUpdater(Optional.of(1));
+ public ParserConfigurations getConfigurations() {
+ return parserConfigurations;
}
};
- parserBolt.setCuratorFramework(client);
- parserBolt.setZKCache(cache);
- parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
- verify(parser, times(1)).init();
- verify(batchWriter, times(1)).init(any(), any(), any());
- when(parser.validate(any())).thenReturn(true);
- when(parser.parseOptionalResult(any())).thenReturn(Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject()))));
- when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
- BulkWriterResponse response = new BulkWriterResponse();
- response.addSuccess(t1);
- when(batchWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(Collections.singleton(t1)), any())).thenReturn(response);
+ parserBolt.setMessageGetStrategy(messageGetStrategy);
+ parserBolt.setOutputCollector(outputCollector);
+ parserBolt.setTopicToSensorMap(new HashMap<String, String>() {{
+ put("yafTopic", "yaf");
+ }});
+ JSONObject message = new JSONObject();
+ message.put("field", "value");
+ mockParserRunner.setMessage(message);
+ RawMessage expectedRawMessage = new RawMessage("originalMessage".getBytes(StandardCharsets.UTF_8), new HashMap<>());
+ MetronError error = new MetronError()
+ .withErrorType(Constants.ErrorType.PARSER_INVALID)
+ .withSensorType(Collections.singleton("yaf"))
+ .addRawMessage(message);
+
parserBolt.execute(t1);
+
+ Assert.assertEquals(expectedRawMessage, mockParserRunner.getRawMessage());
+ verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM),
+ argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
verify(outputCollector, times(1)).ack(t1);
+
}
@Test
- public void testBatchOfFive() throws Exception {
- String sensorType = "yaf";
- Map<String, ParserComponents> parserMap = Collections.singletonMap(
- sensorType,
- new ParserComponents(
- parser,
- filter,
- new WriterHandler(batchWriter)
- )
- );
- ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserMap) {
- @Override
- protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
- return ParserBoltTest.createUpdater(Optional.of(5));
- }
- } ;
-
- parserBolt.setCuratorFramework(client);
- parserBolt.setZKCache(cache);
- parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
- verify(parser, times(1)).init();
- verify(batchWriter, times(1)).init(any(), any(), any());
- when(parser.validate(any())).thenReturn(true);
- when(parser.parseOptionalResult(any())).thenReturn(Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject()))));
- when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
- Set<Tuple> tuples = Stream.of(t1, t2, t3, t4, t5).collect(Collectors.toSet());
- BulkWriterResponse response = new BulkWriterResponse();
- response.addAllSuccesses(tuples);
- when(batchWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(tuples), any())).thenReturn(response);
- writeNonBatch(outputCollector, parserBolt, t1);
- writeNonBatch(outputCollector, parserBolt, t2);
- writeNonBatch(outputCollector, parserBolt, t3);
- writeNonBatch(outputCollector, parserBolt, t4);
- parserBolt.execute(t5);
- verify(batchWriter, times(1)).write(eq(sensorType), any(WriterConfiguration.class), eq(tuples), any());
- verify(outputCollector, times(1)).ack(t1);
- verify(outputCollector, times(1)).ack(t2);
- verify(outputCollector, times(1)).ack(t3);
- verify(outputCollector, times(1)).ack(t4);
- verify(outputCollector, times(1)).ack(t5);
+ public void shouldThrowExceptionOnFailedExecute() {
+ when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
+ when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
+ ParserConfigurations parserConfigurations = new ParserConfigurations();
+ parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
+ doThrow(new IllegalStateException("parserRunner.execute failed")).when(parserRunner).execute(eq("yaf"), any(), eq(parserConfigurations));
- }
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserRunner, new HashMap<String, WriterHandler>() {{
+ put("yaf", writerHandler);
+ }}) {
- @Test
- public void testBatchOfFiveWithError() throws Exception {
- String sensorType = "yaf";
- Map<String, ParserComponents> parserMap = Collections.singletonMap(
- sensorType,
- new ParserComponents(
- parser,
- filter,
- new WriterHandler(batchWriter)
- )
- );
- ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserMap) {
@Override
- protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
- return ParserBoltTest.createUpdater(Optional.of(5));
+ public ParserConfigurations getConfigurations() {
+ return parserConfigurations;
}
};
- parserBolt.setCuratorFramework(client);
- parserBolt.setZKCache(cache);
- parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
- verify(parser, times(1)).init();
- verify(batchWriter, times(1)).init(any(), any(), any());
-
- doThrow(new Exception()).when(batchWriter).write(any(), any(), any(), any());
- when(parser.validate(any())).thenReturn(true);
- when(parser.parseOptionalResult(any())).thenReturn(Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject()))));
- when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
- parserBolt.execute(t1);
- parserBolt.execute(t2);
- parserBolt.execute(t3);
- parserBolt.execute(t4);
- parserBolt.execute(t5);
- verify(batchWriter, times(1)).write(any(), any(), any(), any());
- verify(outputCollector, times(1)).ack(t1);
- verify(outputCollector, times(1)).ack(t2);
- verify(outputCollector, times(1)).ack(t3);
- verify(outputCollector, times(1)).ack(t4);
- verify(outputCollector, times(1)).ack(t5);
+ parserBolt.setMessageGetStrategy(messageGetStrategy);
+ parserBolt.setOutputCollector(outputCollector);
+ parserBolt.setTopicToSensorMap(new HashMap<String, String>() {{
+ put("yafTopic", "yaf");
+ }});
+ MetronError error = new MetronError()
+ .withErrorType(Constants.ErrorType.PARSER_ERROR)
+ .withThrowable(new IllegalStateException("parserRunner.execute failed"))
+ .withSensorType(Collections.singleton("yaf"))
+ .addRawMessage("originalMessage".getBytes(StandardCharsets.UTF_8));
- }
+ parserBolt.execute(t1);
- protected void buildGlobalConfig(ParserBolt parserBolt) {
- HashMap<String, Object> globalConfig = new HashMap<>();
- Map<String, Object> fieldValidation = new HashMap<>();
- fieldValidation.put("input", Arrays.asList("field"));
- fieldValidation.put("validation", "STELLAR");
- fieldValidation.put("config", new HashMap<String, String>(){{ put("condition", "field != 'invalidValue'"); }});
- globalConfig.put("fieldValidations", Arrays.asList(fieldValidation));
- parserBolt.getConfigurations().updateGlobalConfig(globalConfig);
+ verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM),
+ argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
+ verify(outputCollector, times(1)).reportError(any(IllegalStateException.class));
+ verify(outputCollector, times(1)).ack(t1);
}
- private ParserBolt buildParserBolt(Map<String, ParserComponents> parserMap,
- String csvWithFieldTransformations) {
- return new ParserBolt("zookeeperUrl", parserMap) {
- @Override
- protected SensorParserConfig getSensorParserConfig(String sensorType) {
- try {
- return SensorParserConfig.fromBytes(Bytes.toBytes(csvWithFieldTransformations));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
+ @Test
+ public void shouldThrowExceptionOnFailedWrite() throws Exception {
+ when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
+ when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
+ MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {{ add("yaf"); }});
+ ParserConfigurations parserConfigurations = new ParserConfigurations();
+ parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
+ doThrow(new IllegalStateException("write failed")).when(writerHandler).write(any(), any(), any(), any(), any());
+
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {{
+ put("yaf", writerHandler);
+ }}) {
@Override
- protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
- return ParserBoltTest.createUpdater(Optional.of(1));
+ public ParserConfigurations getConfigurations() {
+ return parserConfigurations;
}
};
- }
- private static void writeNonBatch(OutputCollector collector, ParserBolt bolt, Tuple t) {
- bolt.execute(t);
- }
+ parserBolt.setMessageGetStrategy(messageGetStrategy);
+ parserBolt.setOutputCollector(outputCollector);
+ parserBolt.setTopicToSensorMap(new HashMap<String, String>() {{
+ put("yafTopic", "yaf");
+ }});
+ JSONObject message = new JSONObject();
+ message.put("field", "value");
+ mockParserRunner.setMessage(message);
+
+ MetronError error = new MetronError()
+ .withErrorType(Constants.ErrorType.PARSER_ERROR)
+ .withThrowable(new IllegalStateException("write failed"))
+ .withSensorType(Collections.singleton("yaf"))
+ .addRawMessage("originalMessage".getBytes(StandardCharsets.UTF_8));
+ parserBolt.execute(t1);
+
+ verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM),
+ argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
+ verify(outputCollector, times(1)).reportError(any(IllegalStateException.class));
+ verify(outputCollector, times(1)).ack(t1);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
index 0d6eef8..31d87a7 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
@@ -25,21 +25,20 @@ import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
+
import org.apache.commons.lang.SerializationUtils;
-import org.apache.metron.common.configuration.FieldValidator;
import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.common.utils.ReflectionUtils;
import org.apache.metron.common.writer.MessageWriter;
import org.apache.metron.integration.ProcessorResult;
+import org.apache.metron.parsers.ParserRunner;
+import org.apache.metron.parsers.ParserRunnerImpl;
import org.apache.metron.parsers.bolt.ParserBolt;
import org.apache.metron.parsers.bolt.WriterHandler;
-import org.apache.metron.parsers.topology.ParserComponents;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
import org.json.simple.JSONObject;
@@ -83,41 +82,13 @@ public class ParserDriver implements Serializable {
List<byte[]> errors = new ArrayList<>();
public ShimParserBolt(List<byte[]> output) {
- super(null
- , Collections.singletonMap(
- sensorType == null ? config.getSensorTopic() : sensorType,
- new ParserComponents(
- ReflectionUtils.createInstance(config.getParserClassName()),
- null,
- new WriterHandler(new CollectingWriter(output))
- )
- )
- );
+ super(null, parserRunner, Collections.singletonMap(sensorType, new WriterHandler(new CollectingWriter(output))));
this.output = output;
- Map<String, ParserComponents> sensorToComponentMap = getSensorToComponentMap();
- for(Entry<String, ParserComponents> sensorToComponents : sensorToComponentMap.entrySet()) {
- sensorToComponents.getValue().getMessageParser().configure(config.getParserConfig());
- }
}
@Override
public ParserConfigurations getConfigurations() {
- return new ParserConfigurations() {
- @Override
- public SensorParserConfig getSensorParserConfig(String sensorType) {
- return config;
- }
-
- @Override
- public Map<String, Object> getGlobalConfig() {
- return globalConfig;
- }
-
- @Override
- public List<FieldValidator> getFieldValidations() {
- return new ArrayList<>();
- }
- };
+ return config;
}
@Override
@@ -125,7 +96,7 @@ public class ParserDriver implements Serializable {
}
@Override
- protected void handleError(byte[] originalMessage, Tuple tuple, Throwable ex, OutputCollector collector) {
+ protected void handleError(String sensorType, byte[] originalMessage, Tuple tuple, Throwable ex, OutputCollector collector) {
errors.add(originalMessage);
LOG.error("Error parsing message: " + ex.getMessage(), ex);
}
@@ -139,14 +110,20 @@ public class ParserDriver implements Serializable {
}
- private SensorParserConfig config;
- private Map<String, Object> globalConfig;
+ private ParserConfigurations config;
private String sensorType;
+ private ParserRunner parserRunner;
public ParserDriver(String sensorType, String parserConfig, String globalConfig) throws IOException {
- config = SensorParserConfig.fromBytes(parserConfig.getBytes());
- this.sensorType = sensorType;
- this.globalConfig = JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER);
+ SensorParserConfig sensorParserConfig = SensorParserConfig.fromBytes(parserConfig.getBytes());
+ this.sensorType = sensorType == null ? sensorParserConfig.getSensorTopic() : sensorType;
+ config = new ParserConfigurations();
+ config.updateSensorParserConfig(this.sensorType, SensorParserConfig.fromBytes(parserConfig.getBytes()));
+ config.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER));
+
+ parserRunner = new ParserRunnerImpl(new HashSet<String>() {{
+ add(sensorType);
+ }});
}
public ProcessorResult<List<byte[]>> run(Iterable<byte[]> in) {
@@ -163,6 +140,7 @@ public class ParserDriver implements Serializable {
public Tuple toTuple(byte[] record) {
Tuple ret = mock(Tuple.class);
+ when(ret.getStringByField("topic")).thenReturn(sensorType);
when(ret.getBinary(eq(0))).thenReturn(record);
return ret;
}
[15/51] [abbrv] metron git commit: METRON-1784: Re-allow remote ssh
and scp in Centos full dev (mmiklavc via mmiklavc) closes apache/metron#1204
Posted by ot...@apache.org.
METRON-1784: Re-allow remote ssh and scp in Centos full dev (mmiklavc via mmiklavc) closes apache/metron#1204
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/4f0b608d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/4f0b608d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/4f0b608d
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 4f0b608d77ebed3025ba0accedec440b3270cb18
Parents: 08fe1cf
Author: mmiklavc <mi...@gmail.com>
Authored: Wed Sep 26 16:34:13 2018 -0600
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Wed Sep 26 16:34:13 2018 -0600
----------------------------------------------------------------------
.../roles/enable-remote-ssh/defaults/main.yml | 18 +++++++++++
.../roles/enable-remote-ssh/tasks/main.yml | 33 ++++++++++++++++++++
.../development/centos6/ansible/playbook.yml | 5 +--
3 files changed, 54 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/4f0b608d/metron-deployment/ansible/roles/enable-remote-ssh/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/enable-remote-ssh/defaults/main.yml b/metron-deployment/ansible/roles/enable-remote-ssh/defaults/main.yml
new file mode 100644
index 0000000..f176095
--- /dev/null
+++ b/metron-deployment/ansible/roles/enable-remote-ssh/defaults/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+sshd_config_file: /etc/ssh/sshd_config
http://git-wip-us.apache.org/repos/asf/metron/blob/4f0b608d/metron-deployment/ansible/roles/enable-remote-ssh/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/enable-remote-ssh/tasks/main.yml b/metron-deployment/ansible/roles/enable-remote-ssh/tasks/main.yml
new file mode 100644
index 0000000..be83aa2
--- /dev/null
+++ b/metron-deployment/ansible/roles/enable-remote-ssh/tasks/main.yml
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Comment password auth no
+ lineinfile:
+ dest: "{{ sshd_config_file }}"
+ regexp: "^PasswordAuthentication no"
+ line: "#PasswordAuthentication no"
+
+- name: Uncomment password auth yes
+ lineinfile:
+ dest: "{{ sshd_config_file }}"
+ regexp: "^#PasswordAuthentication yes"
+ line: "PasswordAuthentication yes"
+
+- name: Restart service sshd, in all cases
+ service:
+ name: sshd
+ state: restarted
http://git-wip-us.apache.org/repos/asf/metron/blob/4f0b608d/metron-deployment/development/centos6/ansible/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/ansible/playbook.yml b/metron-deployment/development/centos6/ansible/playbook.yml
index fa14677..562fb1d 100644
--- a/metron-deployment/development/centos6/ansible/playbook.yml
+++ b/metron-deployment/development/centos6/ansible/playbook.yml
@@ -17,7 +17,8 @@
---
- hosts: all
roles:
- - role: libselinux-python
- - role: enable-swap
+ - { role: libselinux-python, tags: libselinux-python }
+ - { role: enable-swap, tags: enable-swap }
+ - { role: enable-remote-ssh, tags: enable-remote-ssh }
- include: ../../../ansible/playbooks/metron_full_install.yml
[35/51] [abbrv] metron git commit: METRON-1813 Stellar REPL Not
Initialized with Client JAAS (nickwallen) closes apache/metron#1232
Posted by ot...@apache.org.
METRON-1813 Stellar REPL Not Initialized with Client JAAS (nickwallen) closes apache/metron#1232
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/060d17e9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/060d17e9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/060d17e9
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 060d17e992a73bc5b65bb9a52d044a3f672ba2c1
Parents: 1f1b9cd
Author: nickwallen <ni...@nickallen.org>
Authored: Wed Oct 10 09:54:21 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Wed Oct 10 09:54:21 2018 -0400
----------------------------------------------------------------------
metron-platform/metron-common/src/main/scripts/stellar | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/060d17e9/metron-platform/metron-common/src/main/scripts/stellar
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/scripts/stellar b/metron-platform/metron-common/src/main/scripts/stellar
index c831c62..34097f2 100644
--- a/metron-platform/metron-common/src/main/scripts/stellar
+++ b/metron-platform/metron-common/src/main/scripts/stellar
@@ -28,10 +28,15 @@ elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
. /usr/lib/bigtop-utils/bigtop-detect-javahome
fi
+export METRON_SYSCONFIG="/etc/default/metron"
+if [ -f "$METRON_SYSCONFIG" ]; then
+ source $METRON_SYSCONFIG
+fi
+
+export METRON_VERSION="${METRON_VERSION:-${project.version}}"
+export METRON_HOME="${METRON_HOME:-/usr/metron/$METRON_VERSION}"
export HBASE_CONFIGS=$(hbase classpath)
-export METRON_VERSION=${project.version}
-export METRON_HOME=/usr/metron/$METRON_VERSION
export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar)
export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar)
export PROFILER_LIB=$(find $METRON_HOME/lib/ -name metron-profiler-repl*.jar)
-java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$PROFILER_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
+java $METRON_JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$PROFILER_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
[12/51] [abbrv] metron git commit: METRON-1786 Pcap Topology Status
Incorrect (MohanDV via nickwallen) closes apache/metron#1206
Posted by ot...@apache.org.
METRON-1786 Pcap Topology Status Incorrect (MohanDV via nickwallen) closes apache/metron#1206
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0d34e77c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0d34e77c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0d34e77c
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 0d34e77cdb0c9104c848808800158492620c7d9b
Parents: 15194c3
Author: MohanDV <mo...@gmail.com>
Authored: Fri Sep 21 09:41:23 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Fri Sep 21 09:41:23 2018 -0400
----------------------------------------------------------------------
.../METRON/CURRENT/package/scripts/params/status_params.py | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/0d34e77c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 2c711cf..1ca29b2 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -119,6 +119,8 @@ metron_principal_name = config['configurations']['metron-env']['metron_principal
metron_keytab_path = config['configurations']['metron-env']['metron_service_keytab']
# Pcap
+metron_pcap_topology = 'pcap'
+pcap_input_topic = 'pcap'
pcap_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_configured'
pcap_perm_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_perm_configured'
pcap_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_acl_configured'
[02/51] [abbrv] metron git commit: Merge remote-tracking branch
'apache/master' into feature/METRON-1699-create-batch-profiler
Posted by ot...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-storm/src/test/resources/telemetry.json
----------------------------------------------------------------------
diff --cc metron-analytics/metron-profiler-storm/src/test/resources/telemetry.json
index 0000000,0000000..4a324cf
new file mode 100644
--- /dev/null
+++ b/metron-analytics/metron-profiler-storm/src/test/resources/telemetry.json
@@@ -1,0 -1,0 +1,100 @@@
++{"adapter.threatinteladapter.end.ts":"1530978697769","qclass_name":"qclass-32769","bro_timestamp":"1530978687.836793","qtype_name":"PTR","ip_dst_port":5353,"enrichmentsplitterbolt.splitter.end.ts":"1530978696551","qtype":12,"rejected":false,"enrichmentsplitterbolt.splitter.begin.ts":"1530978696550","adapter.hostfromjsonlistadapter.end.ts":"1530978696606","trans_id":0,"adapter.geoadapter.begin.ts":"1530978696857","uid":"CGs8rS1rqhyXRRgA64","protocol":"dns","original_string":"DNS | AA:false qclass_name:qclass-32769 id.orig_p:5353 qtype_name:PTR qtype:12 rejected:false id.resp_p:5353 query:_googlecast._tcp.local trans_id:0 TC:false RA:false uid:CGs8rS1rqhyXRRgA64 RD:false proto:udp id.orig_h:192.168.66.1 Z:0 qclass:32769 ts:1530978687.836793 id.resp_h:224.0.0.251","ip_dst_addr":"224.0.0.251","threatinteljoinbolt.joiner.ts":"1530978697808","enrichmentjoinbolt.joiner.ts":"1530978696932","adapter.hostfromjsonlistadapter.begin.ts":"1530978696606","threatintelsplitterbolt.splitter.begin.ts
":"1530978696949","Z":0,"ip_src_addr":"192.168.66.1","qclass":32769,"timestamp":1530978687836,"AA":false,"query":"_googlecast._tcp.local","TC":false,"RA":false,"source.type":"bro","adapter.geoadapter.end.ts":"1530978696857","RD":false,"threatintelsplitterbolt.splitter.end.ts":"1530978696952","adapter.threatinteladapter.begin.ts":"1530978697764","ip_src_port":5353,"proto":"udp","guid":"90751ce5-703d-4b9f-8c2d-8e5c42e72262"}
++{"adapter.threatinteladapter.end.ts":"1530978697772","bro_timestamp":"1530978687.77394","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696605","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978696605","adapter.hostfromjsonlistadapter.end.ts":"1530978696649","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978696857","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"CBJatv2DcsW8fow3Dg","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49186 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:CBJatv2DcsW8fow3Dg referrer:http:\/\/va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in\/?285a4d4e4e5a4d4d4649584c5d43064b4745 resp_mime_types:[\"text\\\/html\"] trans_depth:1 ho
st:r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in status_msg:OK id.orig_h:192.168.138.158 response_body_len:121635 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978687.77394 id.resp_h:62.75.195.236 resp_fuids:[\"F77a061yn9H0cUBGVa\"]","ip_dst_addr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978697808","host":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","enrichmentjoinbolt.joiner.ts":"1530978696943","adapter.hostfromjsonlistadapter.begin.ts":"1530978696607","threatintelsplitterbolt.splitter.begin.ts":"1530978696952","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":[
"F77a061yn9H0cUBGVa"],"timestamp":1530978687773,"method":"GET","request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696858","referrer":"http:\/\/va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in\/?285a4d4e4e5a4d4d4649584c5d43064b4745","threatintelsplitterbolt.splitter.end.ts":"1530978696952","adapter.threatinteladapter.begin.ts":"1530978697769","ip_src_port":49186,"enrichments.geo.ip_dst_addr.location_point":"48.5839,7.7455","status_msg":"OK","guid":"f5b315b0-e776-481a-9f28-765fdb19e6e8","response_body_len":121635}
++{"adapter.threatinteladapter.end.ts":"1530978697776","bro_timestamp":"1530978687.916811","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978696606","enrichmentsplitterbolt.splitter.begin.ts":"1530978696606","adapter.hostfromjsonlistadapter.end.ts":"1530978696650","adapter.geoadapter.begin.ts":"1530978696858","uid":"CUrRne3iLIxXavQtci","trans_depth":6,"protocol":"http","original_string":"HTTP | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 uri:\/api\/v1\/clusters\/metron_cluster\/components\/?ServiceComponentInfo\/component_name=APP_TIMELINE_SERVER|ServiceComponentInfo\/category=MASTER&fields=ServiceComponentInfo\/service_name,host_components\/HostRoles\/display_name,host_components\/HostRoles\/host_name,host_components\/HostRoles\/state,host_components\/HostRoles\/maintenance_state,host_components\/HostRoles\/stale_configs,host_components\/HostRoles\/ha_state,host_components\/HostRoles\/desired_admin_state,,host_components\/metrics\/jvm\/memHeapUsedM
,host_components\/metrics\/jvm\/HeapMemoryMax,host_components\/metrics\/jvm\/HeapMemoryUsed,host_components\/metrics\/jvm\/memHeapCommittedM,host_components\/metrics\/mapred\/jobtracker\/trackers_decommissioned,host_components\/metrics\/cpu\/cpu_wio,host_components\/metrics\/rpc\/client\/RpcQueueTime_avg_time,host_components\/metrics\/dfs\/FSNamesystem\/*,host_components\/metrics\/dfs\/namenode\/Version,host_components\/metrics\/dfs\/namenode\/LiveNodes,host_components\/metrics\/dfs\/namenode\/DeadNodes,host_components\/metrics\/dfs\/namenode\/DecomNodes,host_components\/metrics\/dfs\/namenode\/TotalFiles,host_components\/metrics\/dfs\/namenode\/UpgradeFinalized,host_components\/metrics\/dfs\/namenode\/Safemode,host_components\/metrics\/runtime\/StartTime,host_components\/metrics\/hbase\/master\/IsActiveMaster,host_components\/metrics\/hbase\/master\/MasterStartTime,host_components\/metrics\/hbase\/master\/MasterActiveTime,host_components\/metrics\/hbase\/master\/AverageLoad,host_co
mponents\/metrics\/master\/AssignmentManger\/ritCount,metrics\/api\/v1\/cluster\/summary,metrics\/api\/v1\/topology\/summary,metrics\/api\/v1\/nimbus\/summary,host_components\/metrics\/yarn\/Queue,host_components\/metrics\/yarn\/ClusterMetrics\/NumActiveNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumLostNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumUnhealthyNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumRebootedNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumDecommissionedNMs&minimal_response=true&_=1484168361295 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/ trans_depth:6 host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36 ts:1530978687.916811 id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1530978697808","host":"node1","enrichmentjoinbolt.joiner.ts":"1530978
696948","adapter.hostfromjsonlistadapter.begin.ts":"1530978696649","threatintelsplitterbolt.splitter.begin.ts":"1530978696953","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36","timestamp":1530978687916,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/clusters\/metron_cluster\/components\/?ServiceComponentInfo\/component_name=APP_TIMELINE_SERVER|ServiceComponentInfo\/category=MASTER&fields=ServiceComponentInfo\/service_name,host_components\/HostRoles\/display_name,host_components\/HostRoles\/host_name,host_components\/HostRoles\/state,host_components\/HostRoles\/maintenance_state,host_components\/HostRoles\/stale_configs,host_components\/HostRoles\/ha_state,host_components\/HostRoles\/desired_admin_state,,host_components\/metrics\/jvm\/memHeapUsedM,host_components\/metrics\/jvm\/HeapMemoryMax,host_components\/metrics\/jvm\/HeapMemoryUsed,host_components\/metric
s\/jvm\/memHeapCommittedM,host_components\/metrics\/mapred\/jobtracker\/trackers_decommissioned,host_components\/metrics\/cpu\/cpu_wio,host_components\/metrics\/rpc\/client\/RpcQueueTime_avg_time,host_components\/metrics\/dfs\/FSNamesystem\/*,host_components\/metrics\/dfs\/namenode\/Version,host_components\/metrics\/dfs\/namenode\/LiveNodes,host_components\/metrics\/dfs\/namenode\/DeadNodes,host_components\/metrics\/dfs\/namenode\/DecomNodes,host_components\/metrics\/dfs\/namenode\/TotalFiles,host_components\/metrics\/dfs\/namenode\/UpgradeFinalized,host_components\/metrics\/dfs\/namenode\/Safemode,host_components\/metrics\/runtime\/StartTime,host_components\/metrics\/hbase\/master\/IsActiveMaster,host_components\/metrics\/hbase\/master\/MasterStartTime,host_components\/metrics\/hbase\/master\/MasterActiveTime,host_components\/metrics\/hbase\/master\/AverageLoad,host_components\/metrics\/master\/AssignmentManger\/ritCount,metrics\/api\/v1\/cluster\/summary,metrics\/api\/v1\/topology
\/summary,metrics\/api\/v1\/nimbus\/summary,host_components\/metrics\/yarn\/Queue,host_components\/metrics\/yarn\/ClusterMetrics\/NumActiveNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumLostNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumUnhealthyNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumRebootedNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumDecommissionedNMs&minimal_response=true&_=1484168361295","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696858","referrer":"http:\/\/node1:8080\/","threatintelsplitterbolt.splitter.end.ts":"1530978696953","adapter.threatinteladapter.begin.ts":"1530978697772","ip_src_port":50451,"guid":"db5e7329-9439-4a8a-972b-05d22d08e1fa","response_body_len":0}
++{"adapter.threatinteladapter.end.ts":"1530978697777","bro_timestamp":"1530978687.073175","status_code":404,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696609","enrichments.geo.ip_dst_addr.city":"Phoenix","enrichments.geo.ip_dst_addr.latitude":"33.4499","enrichmentsplitterbolt.splitter.begin.ts":"1530978696609","adapter.hostfromjsonlistadapter.end.ts":"1530978696650","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5308655","adapter.geoadapter.begin.ts":"1530978696858","enrichments.geo.ip_dst_addr.postalCode":"85004","uid":"CxQY13LFLIWBK5kw6","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49203 status_code:404 method:POST request_body_len:110 id.resp_p:80 orig_mime_types:[\"text\\\/plain\"] uri:\/wp-content\/themes\/twentyfifteen\/img5.php?f=ka6nnuvccqlw9 tags:[] uid:CxQY13LFLIWBK5kw6 resp_mime_types:[\"text\\\/html\"] trans_depth:1 orig_fuids:[\"FUF7cQ2NWtIJObUXFf\"] host:ru
nlove.us status_msg:Not Found id.orig_h:192.168.138.158 response_body_len:357 user_agent:Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978687.073175 id.resp_h:204.152.254.221 resp_fuids:[\"FNXPE1PFFrR89EeJa\"]","ip_dst_addr":"204.152.254.221","threatinteljoinbolt.joiner.ts":"1530978697808","enrichments.geo.ip_dst_addr.dmaCode":"753","host":"runlove.us","enrichmentjoinbolt.joiner.ts":"1530978696948","adapter.hostfromjsonlistadapter.begin.ts":"1530978696650","threatintelsplitterbolt.splitter.begin.ts":"1530978696953","enrichments.geo.ip_dst_addr.longitude":"-112.0712","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FNXPE1PFFrR89EeJa"],"timestamp":1530978687073,"method":"POST","request_body_len
":110,"orig_mime_types":["text\/plain"],"uri":"\/wp-content\/themes\/twentyfifteen\/img5.php?f=ka6nnuvccqlw9","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696870","threatintelsplitterbolt.splitter.end.ts":"1530978696953","adapter.threatinteladapter.begin.ts":"1530978697776","orig_fuids":["FUF7cQ2NWtIJObUXFf"],"ip_src_port":49203,"enrichments.geo.ip_dst_addr.location_point":"33.4499,-112.0712","status_msg":"Not Found","guid":"1d9eefeb-832b-4262-a800-5b67da9f7277","response_body_len":357}
++{"adapter.threatinteladapter.end.ts":"1530978697780","bro_timestamp":"1530978687.027914","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696609","enrichments.geo.ip_dst_addr.city":"Los Angeles","enrichments.geo.ip_dst_addr.latitude":"34.0494","enrichmentsplitterbolt.splitter.begin.ts":"1530978696609","adapter.hostfromjsonlistadapter.end.ts":"1530978696651","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5368361","adapter.geoadapter.begin.ts":"1530978696870","enrichments.geo.ip_dst_addr.postalCode":"90014","uid":"CxZIVD4f5vBwpXUjwf","resp_mime_types":["text\/plain"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49198 status_code:200 method:POST request_body_len:134 id.resp_p:80 orig_mime_types:[\"text\\\/plain\"] uri:\/wp-content\/themes\/grizzly\/img5.php?c=cdcnw7cfz43rmtg tags:[] uid:CxZIVD4f5vBwpXUjwf resp_mime_types:[\"text\\\/plain\"] trans_depth:1 orig_fuids:[\"FiPZ8g4gdpjEyHuez2\"] hos
t:comarksecurity.com status_msg:OK id.orig_h:192.168.138.158 response_body_len:14 user_agent:Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978687.027914 id.resp_h:72.34.49.86 resp_fuids:[\"FM8l2i6ib3vOd45ob\"]","ip_dst_addr":"72.34.49.86","threatinteljoinbolt.joiner.ts":"1530978697808","enrichments.geo.ip_dst_addr.dmaCode":"803","host":"comarksecurity.com","enrichmentjoinbolt.joiner.ts":"1530978696949","adapter.hostfromjsonlistadapter.begin.ts":"1530978696650","threatintelsplitterbolt.splitter.begin.ts":"1530978696953","enrichments.geo.ip_dst_addr.longitude":"-118.2641","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FM8l2i6ib3vOd45ob"],"timestamp":1530978687027,"method":"POST","request_body
_len":134,"orig_mime_types":["text\/plain"],"uri":"\/wp-content\/themes\/grizzly\/img5.php?c=cdcnw7cfz43rmtg","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696875","threatintelsplitterbolt.splitter.end.ts":"1530978696953","adapter.threatinteladapter.begin.ts":"1530978697778","orig_fuids":["FiPZ8g4gdpjEyHuez2"],"ip_src_port":49198,"enrichments.geo.ip_dst_addr.location_point":"34.0494,-118.2641","status_msg":"OK","guid":"0c21f313-5cb7-46de-b62a-b429c565bfb0","response_body_len":14}
++{"adapter.threatinteladapter.end.ts":"1530978697782","bro_timestamp":"1530978687.58302","ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696609","enrichments.geo.ip_dst_addr.latitude":"48.8582","enrichmentsplitterbolt.splitter.begin.ts":"1530978696609","adapter.hostfromjsonlistadapter.end.ts":"1530978696651","enrichments.geo.ip_dst_addr.country":"FR","adapter.geoadapter.begin.ts":"1530978696875","uid":"CT2ax04BCxPW20AlGc","trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49195 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:CT2ax04BCxPW20AlGc trans_depth:1 host:ip-addr.es id.orig_h:192.168.138.158 response_body_len:0 user_agent:Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978687.58302 id.resp_h:188.165.164.184","ip_dst_addr":"188.165.164.184","threatinteljoinbolt.joiner.ts":"1530978697809","host":"ip-addr.es","
enrichmentjoinbolt.joiner.ts":"1530978696949","adapter.hostfromjsonlistadapter.begin.ts":"1530978696651","threatintelsplitterbolt.splitter.begin.ts":"1530978696953","enrichments.geo.ip_dst_addr.longitude":"2.3387000000000002","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","timestamp":1530978687583,"method":"GET","request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696879","threatintelsplitterbolt.splitter.end.ts":"1530978696953","adapter.threatinteladapter.begin.ts":"1530978697780","ip_src_port":49195,"enrichments.geo.ip_dst_addr.location_point":"48.8582,2.3387000000000002","guid":"ed0d58c1-88cb-4f4e-ab7b-ee1a36e7cdcb","response_body_len":0}
++{"adapter.threatinteladapter.end.ts":"1530978697783","qclass_name":"C_INTERNET","bro_timestamp":"1530978687.445971","qtype_name":"PTR","ip_dst_port":5353,"enrichmentsplitterbolt.splitter.end.ts":"1530978696610","qtype":12,"rejected":false,"enrichmentsplitterbolt.splitter.begin.ts":"1530978696610","adapter.hostfromjsonlistadapter.end.ts":"1530978696651","trans_id":0,"adapter.geoadapter.begin.ts":"1530978696879","uid":"ChMDrL20pLP4UzCncj","protocol":"dns","original_string":"DNS | AA:false qclass_name:C_INTERNET id.orig_p:5353 qtype_name:PTR qtype:12 rejected:false id.resp_p:5353 query:_googlecast._tcp.local trans_id:0 TC:false RA:false uid:ChMDrL20pLP4UzCncj RD:false proto:udp id.orig_h:192.168.66.1 Z:0 qclass:1 ts:1530978687.445971 id.resp_h:224.0.0.251","ip_dst_addr":"224.0.0.251","threatinteljoinbolt.joiner.ts":"1530978697809","enrichmentjoinbolt.joiner.ts":"1530978696949","adapter.hostfromjsonlistadapter.begin.ts":"1530978696651","threatintelsplitterbolt.splitter.begin.ts":"15309
78696953","Z":0,"ip_src_addr":"192.168.66.1","qclass":1,"timestamp":1530978687445,"AA":false,"query":"_googlecast._tcp.local","TC":false,"RA":false,"source.type":"bro","adapter.geoadapter.end.ts":"1530978696879","RD":false,"threatintelsplitterbolt.splitter.end.ts":"1530978696953","adapter.threatinteladapter.begin.ts":"1530978697783","ip_src_port":5353,"proto":"udp","guid":"a6f4fe3a-c485-4521-bcfe-b2600746885e"}
++{"TTLs":[21599.0],"adapter.threatinteladapter.end.ts":"1530978697784","qclass_name":"C_INTERNET","bro_timestamp":"1530978687.053752","qtype_name":"A","ip_dst_port":53,"enrichmentsplitterbolt.splitter.end.ts":"1530978696610","qtype":1,"rejected":false,"answers":["188.165.164.184"],"enrichmentsplitterbolt.splitter.begin.ts":"1530978696610","adapter.hostfromjsonlistadapter.end.ts":"1530978696652","trans_id":15553,"adapter.geoadapter.begin.ts":"1530978696879","uid":"CoiTkw2sb9stNr10zg","protocol":"dns","original_string":"DNS | AA:false TTLs:[21599.0] qclass_name:C_INTERNET id.orig_p:53571 qtype_name:A qtype:1 rejected:false id.resp_p:53 query:ip-addr.es answers:[\"188.165.164.184\"] trans_id:15553 rcode:0 rcode_name:NOERROR TC:false RA:true uid:CoiTkw2sb9stNr10zg RD:true proto:udp id.orig_h:192.168.138.158 Z:0 qclass:1 ts:1530978687.053752 id.resp_h:192.168.138.2","ip_dst_addr":"192.168.138.2","threatinteljoinbolt.joiner.ts":"1530978697809","enrichmentjoinbolt.joiner.ts":"1530978696953
","adapter.hostfromjsonlistadapter.begin.ts":"1530978696652","threatintelsplitterbolt.splitter.begin.ts":"1530978696961","Z":0,"ip_src_addr":"192.168.138.158","qclass":1,"timestamp":1530978687053,"AA":false,"query":"ip-addr.es","rcode":0,"rcode_name":"NOERROR","TC":false,"RA":true,"source.type":"bro","adapter.geoadapter.end.ts":"1530978696879","RD":true,"threatintelsplitterbolt.splitter.end.ts":"1530978696961","adapter.threatinteladapter.begin.ts":"1530978697783","ip_src_port":53571,"proto":"udp","guid":"bbfd5e54-db09-455e-b01f-b6cbbd444e88"}
++{"adapter.threatinteladapter.end.ts":"1530978697784","bro_timestamp":"1530978687.267256","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696610","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978696610","adapter.hostfromjsonlistadapter.end.ts":"1530978696652","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978696880","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"CID7qb45BoqLfAMHic","trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49193 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/?34eaf8bd50d85d8c6baacb45f0a7b22e tags:[] uid:CID7qb45BoqLfAMHic trans_depth:1 host:62.75.195.236 status_msg:OK id.orig_h:192.168.138.158 response_body_len:0 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .
NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978687.267256 id.resp_h:62.75.195.236","ip_dst_addr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978697809","host":"62.75.195.236","enrichmentjoinbolt.joiner.ts":"1530978696953","adapter.hostfromjsonlistadapter.begin.ts":"1530978696652","threatintelsplitterbolt.splitter.begin.ts":"1530978696961","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","timestamp":1530978687267,"method":"GET","request_body_len":0,"uri":"\/?34eaf8bd50d85d8c6baacb45f0a7b22e","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696880","threatintelsplitterbolt.splitter.end.ts":"1530978696961","adapter.threatinteladapter.begin.ts":"1530978697784","ip_src_port":49193,"enrichments.geo.ip_dst_addr.locati
on_point":"48.5839,7.7455","status_msg":"OK","guid":"ad2f6714-2a4a-4262-8ce0-1940f3e8f340","response_body_len":0}
++{"adapter.threatinteladapter.end.ts":"1530978697786","bro_timestamp":"1530978687.417086","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696610","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978696610","adapter.hostfromjsonlistadapter.end.ts":"1530978696652","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978696880","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"CEkDUW1JYqnTIkYzc1","resp_mime_types":["image\/png"],"trans_depth":2,"protocol":"http","original_string":"HTTP | id.orig_p:49210 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/lb.png tags:[] uid:CEkDUW1JYqnTIkYzc1 referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:2 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:192.
168.138.158 response_body_len:239 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978687.417086 id.resp_h:95.163.121.204 resp_fuids:[\"FZy6Lx4RGFmQ1AZeU8\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978697809","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978696953","adapter.hostfromjsonlistadapter.begin.ts":"1530978696652","threatintelsplitterbolt.splitter.begin.ts":"1530978696961","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FZy6Lx4RGFmQ1AZeU8"],"timestamp":1530978687417,"method":"GET","request_body_len":0,"uri":"\/img\/lb.png","tags":[],"source.type":"bro","adapter.geoad
apter.end.ts":"1530978696887","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978696961","adapter.threatinteladapter.begin.ts":"1530978697784","ip_src_port":49210,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"9711e04e-9926-4606-a7e8-e719dac535e6","response_body_len":239}
++{"adapter.threatinteladapter.end.ts":"1530978697786","qclass_name":"C_INTERNET","bro_timestamp":"1530978694.884106","qtype_name":"PTR","ip_dst_port":5353,"enrichmentsplitterbolt.splitter.end.ts":"1530978696611","qtype":12,"rejected":false,"enrichmentsplitterbolt.splitter.begin.ts":"1530978696611","adapter.hostfromjsonlistadapter.end.ts":"1530978696652","trans_id":0,"adapter.geoadapter.begin.ts":"1530978696887","uid":"CkwtUK1ANyyZwj0PW1","protocol":"dns","original_string":"DNS | AA:false qclass_name:C_INTERNET id.orig_p:5353 qtype_name:PTR qtype:12 rejected:false id.resp_p:5353 query:_googlecast._tcp.local trans_id:0 TC:false RA:false uid:CkwtUK1ANyyZwj0PW1 RD:false proto:udp id.orig_h:192.168.66.1 Z:0 qclass:1 ts:1530978694.884106 id.resp_h:224.0.0.251","ip_dst_addr":"224.0.0.251","threatinteljoinbolt.joiner.ts":"1530978697809","enrichmentjoinbolt.joiner.ts":"1530978696953","adapter.hostfromjsonlistadapter.begin.ts":"1530978696652","threatintelsplitterbolt.splitter.begin.ts":"15309
78696961","Z":0,"ip_src_addr":"192.168.66.1","qclass":1,"timestamp":1530978694884,"AA":false,"query":"_googlecast._tcp.local","TC":false,"RA":false,"source.type":"bro","adapter.geoadapter.end.ts":"1530978696887","RD":false,"threatintelsplitterbolt.splitter.end.ts":"1530978696961","adapter.threatinteladapter.begin.ts":"1530978697786","ip_src_port":5353,"proto":"udp","guid":"f9c14d84-59c5-4598-97b7-5d6e95aba4e6"}
++{"adapter.threatinteladapter.end.ts":"1530978697786","bro_timestamp":"1530978694.621046","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696611","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978696611","adapter.hostfromjsonlistadapter.end.ts":"1530978696653","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978696887","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"C1ia4w3K4ngOWPmAsi","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49186 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:C1ia4w3K4ngOWPmAsi referrer:http:\/\/va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in\/?285a4d4e4e5a4d4d4649584c5d43064b4745 resp_mime_types:[\"text\\\/html\"] trans_depth:1 h
ost:r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in status_msg:OK id.orig_h:192.168.138.158 response_body_len:121635 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.621046 id.resp_h:62.75.195.236 resp_fuids:[\"FKGK2W1X8Bfk7D7XD9\"]","ip_dst_addr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978697809","host":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","enrichmentjoinbolt.joiner.ts":"1530978696953","adapter.hostfromjsonlistadapter.begin.ts":"1530978696652","threatintelsplitterbolt.splitter.begin.ts":"1530978696962","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids"
:["FKGK2W1X8Bfk7D7XD9"],"timestamp":1530978694621,"method":"GET","request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696887","referrer":"http:\/\/va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in\/?285a4d4e4e5a4d4d4649584c5d43064b4745","threatintelsplitterbolt.splitter.end.ts":"1530978696962","adapter.threatinteladapter.begin.ts":"1530978697786","ip_src_port":49186,"enrichments.geo.ip_dst_addr.location_point":"48.5839,7.7455","status_msg":"OK","guid":"98769db4-ee20-4f69-bb04-2e7005de9c6d","response_body_len":121635}
++{"adapter.threatinteladapter.end.ts":"1530978697787","bro_timestamp":"1530978694.641679","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978696611","enrichmentsplitterbolt.splitter.begin.ts":"1530978696611","adapter.hostfromjsonlistadapter.end.ts":"1530978696653","adapter.geoadapter.begin.ts":"1530978696887","uid":"CUrRne3iLIxXavQtci","trans_depth":254,"protocol":"http","original_string":"HTTP | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 uri:\/api\/v1\/persist\/wizard-data?_=1484169473684 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/ trans_depth:254 host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36 ts:1530978694.641679 id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1530978697810","host":"node1","enrichmentjoinbolt.joiner.ts":"1530978696954","adapter.ho
stfromjsonlistadapter.begin.ts":"1530978696653","threatintelsplitterbolt.splitter.begin.ts":"1530978696962","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36","timestamp":1530978694641,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/persist\/wizard-data?_=1484169473684","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696887","referrer":"http:\/\/node1:8080\/","threatintelsplitterbolt.splitter.end.ts":"1530978696962","adapter.threatinteladapter.begin.ts":"1530978697787","ip_src_port":50451,"guid":"ffda80d8-44e3-42db-b72a-d2fa7cf38042","response_body_len":0}
++{"adapter.threatinteladapter.end.ts":"1530978697788","bro_timestamp":"1530978694.388009","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696611","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978696611","adapter.hostfromjsonlistadapter.end.ts":"1530978696653","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978696887","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"CJyQ1119VSEe7SGiTa","resp_mime_types":["image\/x-icon"],"trans_depth":2,"protocol":"http","original_string":"HTTP | id.orig_p:49207 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/favicon.ico tags:[] uid:CJyQ1119VSEe7SGiTa resp_mime_types:[\"image\\\/x-icon\"] trans_depth:2 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:192.168.138.158 response_body_len:318 user_agent:Mozilla\
/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.388009 id.resp_h:95.163.121.204 resp_fuids:[\"F0Pvmv1dj2gRa9c7v\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978697810","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978696954","adapter.hostfromjsonlistadapter.begin.ts":"1530978696653","threatintelsplitterbolt.splitter.begin.ts":"1530978696962","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["F0Pvmv1dj2gRa9c7v"],"timestamp":1530978694388,"method":"GET","request_body_len":0,"uri":"\/favicon.ico","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696887","threatintelsplitterbolt.
splitter.end.ts":"1530978696962","adapter.threatinteladapter.begin.ts":"1530978697787","ip_src_port":49207,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"66eda80b-7f24-4aec-85b9-e381e128dfc7","response_body_len":318}
++{"adapter.threatinteladapter.end.ts":"1530978697788","bro_timestamp":"1530978694.979947","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696611","enrichments.geo.ip_dst_addr.city":"Los Angeles","enrichments.geo.ip_dst_addr.latitude":"34.0494","enrichmentsplitterbolt.splitter.begin.ts":"1530978696611","adapter.hostfromjsonlistadapter.end.ts":"1530978696653","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5368361","adapter.geoadapter.begin.ts":"1530978696887","enrichments.geo.ip_dst_addr.postalCode":"90014","uid":"COZAhy4ljJ4lBc5bgf","resp_mime_types":["text\/plain"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49204 status_code:200 method:POST request_body_len:110 id.resp_p:80 orig_mime_types:[\"text\\\/plain\"] uri:\/wp-content\/themes\/grizzly\/img5.php?u=ka6nnuvccqlw9 tags:[] uid:COZAhy4ljJ4lBc5bgf resp_mime_types:[\"text\\\/plain\"] trans_depth:1 orig_fuids:[\"FgncKy2eauwZjDL6h9\"] host:
comarksecurity.com status_msg:OK id.orig_h:192.168.138.158 response_body_len:14 user_agent:Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.979947 id.resp_h:72.34.49.86 resp_fuids:[\"FDVxtiyWLP0KeNRg8\"]","ip_dst_addr":"72.34.49.86","threatinteljoinbolt.joiner.ts":"1530978697810","enrichments.geo.ip_dst_addr.dmaCode":"803","host":"comarksecurity.com","enrichmentjoinbolt.joiner.ts":"1530978696954","adapter.hostfromjsonlistadapter.begin.ts":"1530978696653","threatintelsplitterbolt.splitter.begin.ts":"1530978696962","enrichments.geo.ip_dst_addr.longitude":"-118.2641","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FDVxtiyWLP0KeNRg8"],"timestamp":1530978694979,"method":"POST","request_body_l
en":110,"orig_mime_types":["text\/plain"],"uri":"\/wp-content\/themes\/grizzly\/img5.php?u=ka6nnuvccqlw9","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696887","threatintelsplitterbolt.splitter.end.ts":"1530978696962","adapter.threatinteladapter.begin.ts":"1530978697788","orig_fuids":["FgncKy2eauwZjDL6h9"],"ip_src_port":49204,"enrichments.geo.ip_dst_addr.location_point":"34.0494,-118.2641","status_msg":"OK","guid":"a5bc5b67-a861-43e2-9232-fb902239cea3","response_body_len":14}
++{"adapter.threatinteladapter.end.ts":"1530978702605","bro_timestamp":"1530978694.045879","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702588","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702588","adapter.hostfromjsonlistadapter.end.ts":"1530978702593","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702593","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"CmNSa535EEM4iN5uwh","resp_mime_types":["image\/png"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49209 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/flags\/de.png tags:[] uid:CmNSa535EEM4iN5uwh referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:1 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig
_h:192.168.138.158 response_body_len:534 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.045879 id.resp_h:95.163.121.204 resp_fuids:[\"FZgahz2hSOfoAP9y1l\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702609","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702598","adapter.hostfromjsonlistadapter.begin.ts":"1530978702593","threatintelsplitterbolt.splitter.begin.ts":"1530978702601","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FZgahz2hSOfoAP9y1l"],"timestamp":1530978694045,"method":"GET","request_body_len":0,"uri":"\/img\/flags\/de.png","tags":[],"source.type":"bro",
"adapter.geoadapter.end.ts":"1530978702593","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978702601","adapter.threatinteladapter.begin.ts":"1530978702605","ip_src_port":49209,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"9e19e186-6aba-45ad-8b70-9e696ef02448","response_body_len":534}
++{"adapter.threatinteladapter.end.ts":"1530978702605","bro_timestamp":"1530978694.98983","status_code":404,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702589","enrichments.geo.ip_dst_addr.city":"Phoenix","enrichments.geo.ip_dst_addr.latitude":"33.4499","enrichmentsplitterbolt.splitter.begin.ts":"1530978702589","adapter.hostfromjsonlistadapter.end.ts":"1530978702593","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5308655","adapter.geoadapter.begin.ts":"1530978702593","enrichments.geo.ip_dst_addr.postalCode":"85004","uid":"CPbKPD2f2Vg9rvtXXk","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49199 status_code:404 method:POST request_body_len:96 id.resp_p:80 orig_mime_types:[\"text\\\/plain\"] uri:\/wp-content\/themes\/twentyfifteen\/img5.php?l=8r1gf1b2t1kuq42 tags:[] uid:CPbKPD2f2Vg9rvtXXk resp_mime_types:[\"text\\\/html\"] trans_depth:1 orig_fuids:[\"FVYpPq1KmqTn8vOfT\"] host:r
unlove.us status_msg:Not Found id.orig_h:192.168.138.158 response_body_len:357 user_agent:Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.98983 id.resp_h:204.152.254.221 resp_fuids:[\"FnmQJXPDEKwZZ8TMf\"]","ip_dst_addr":"204.152.254.221","threatinteljoinbolt.joiner.ts":"1530978702609","enrichments.geo.ip_dst_addr.dmaCode":"753","host":"runlove.us","enrichmentjoinbolt.joiner.ts":"1530978702598","adapter.hostfromjsonlistadapter.begin.ts":"1530978702593","threatintelsplitterbolt.splitter.begin.ts":"1530978702601","enrichments.geo.ip_dst_addr.longitude":"-112.0712","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FnmQJXPDEKwZZ8TMf"],"timestamp":1530978694989,"method":"POST","request_body_len
":96,"orig_mime_types":["text\/plain"],"uri":"\/wp-content\/themes\/twentyfifteen\/img5.php?l=8r1gf1b2t1kuq42","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978702593","threatintelsplitterbolt.splitter.end.ts":"1530978702601","adapter.threatinteladapter.begin.ts":"1530978702605","orig_fuids":["FVYpPq1KmqTn8vOfT"],"ip_src_port":49199,"enrichments.geo.ip_dst_addr.location_point":"33.4499,-112.0712","status_msg":"Not Found","guid":"23070f86-2358-4f4c-9bf4-a612afc8c3e3","response_body_len":357}
++{"adapter.threatinteladapter.end.ts":"1530978702605","bro_timestamp":"1530978694.665931","status_code":404,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702589","enrichments.geo.ip_dst_addr.city":"Phoenix","enrichments.geo.ip_dst_addr.latitude":"33.4499","enrichmentsplitterbolt.splitter.begin.ts":"1530978702589","adapter.hostfromjsonlistadapter.end.ts":"1530978702593","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5308655","adapter.geoadapter.begin.ts":"1530978702593","enrichments.geo.ip_dst_addr.postalCode":"85004","uid":"CQPUy829Fo1TwbqZh5","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49203 status_code:404 method:POST request_body_len:110 id.resp_p:80 orig_mime_types:[\"text\\\/plain\"] uri:\/wp-content\/themes\/twentyfifteen\/img5.php?f=ka6nnuvccqlw9 tags:[] uid:CQPUy829Fo1TwbqZh5 resp_mime_types:[\"text\\\/html\"] trans_depth:1 orig_fuids:[\"FHf5Gv2fxGeTgj5aLk\"] host:
runlove.us status_msg:Not Found id.orig_h:192.168.138.158 response_body_len:357 user_agent:Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.665931 id.resp_h:204.152.254.221 resp_fuids:[\"FuBgoE4Ro7nr1s5NO8\"]","ip_dst_addr":"204.152.254.221","threatinteljoinbolt.joiner.ts":"1530978702609","enrichments.geo.ip_dst_addr.dmaCode":"753","host":"runlove.us","enrichmentjoinbolt.joiner.ts":"1530978702599","adapter.hostfromjsonlistadapter.begin.ts":"1530978702593","threatintelsplitterbolt.splitter.begin.ts":"1530978702601","enrichments.geo.ip_dst_addr.longitude":"-112.0712","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FuBgoE4Ro7nr1s5NO8"],"timestamp":1530978694665,"method":"POST","request_body
_len":110,"orig_mime_types":["text\/plain"],"uri":"\/wp-content\/themes\/twentyfifteen\/img5.php?f=ka6nnuvccqlw9","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978702593","threatintelsplitterbolt.splitter.end.ts":"1530978702601","adapter.threatinteladapter.begin.ts":"1530978702605","orig_fuids":["FHf5Gv2fxGeTgj5aLk"],"ip_src_port":49203,"enrichments.geo.ip_dst_addr.location_point":"33.4499,-112.0712","status_msg":"Not Found","guid":"41e087a9-84a3-41a2-af03-b0ade87ffa76","response_body_len":357}
++{"adapter.threatinteladapter.end.ts":"1530978702605","bro_timestamp":"1530978694.939958","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702590","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702589","adapter.hostfromjsonlistadapter.end.ts":"1530978702593","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702593","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"CA0G2ASkF1efFirs7","resp_mime_types":["image\/png"],"trans_depth":2,"protocol":"http","original_string":"HTTP | id.orig_p:49210 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/lb.png tags:[] uid:CA0G2ASkF1efFirs7 referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:2 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:192.16
8.138.158 response_body_len:239 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.939958 id.resp_h:95.163.121.204 resp_fuids:[\"FXqalu3YBvkNyrelff\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702610","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702599","adapter.hostfromjsonlistadapter.begin.ts":"1530978702593","threatintelsplitterbolt.splitter.begin.ts":"1530978702601","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FXqalu3YBvkNyrelff"],"timestamp":1530978694939,"method":"GET","request_body_len":0,"uri":"\/img\/lb.png","tags":[],"source.type":"bro","adapter.geoadap
ter.end.ts":"1530978702593","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978702601","adapter.threatinteladapter.begin.ts":"1530978702605","ip_src_port":49210,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"b4a27884-579e-4266-b1d5-0c12f941924a","response_body_len":239}
++{"adapter.threatinteladapter.end.ts":"1530978702605","bro_timestamp":"1530978694.291127","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702590","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702590","adapter.hostfromjsonlistadapter.end.ts":"1530978702593","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702593","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"CodIOCgeqZXqVSCg6","resp_mime_types":["image\/png"],"trans_depth":4,"protocol":"http","original_string":"HTTP | id.orig_p:49205 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/bitcoin.png tags:[] uid:CodIOCgeqZXqVSCg6 referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:4 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:1
92.168.138.158 response_body_len:5523 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.291127 id.resp_h:95.163.121.204 resp_fuids:[\"Ft8inr3vk76ny20gZ2\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702610","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702599","adapter.hostfromjsonlistadapter.begin.ts":"1530978702593","threatintelsplitterbolt.splitter.begin.ts":"1530978702601","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["Ft8inr3vk76ny20gZ2"],"timestamp":1530978694291,"method":"GET","request_body_len":0,"uri":"\/img\/bitcoin.png","tags":[],"source.type":"bro","adap
ter.geoadapter.end.ts":"1530978702593","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978702601","adapter.threatinteladapter.begin.ts":"1530978702605","ip_src_port":49205,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"105bf657-f1ec-4276-bfcd-091905599296","response_body_len":5523}
++{"adapter.threatinteladapter.end.ts":"1530978702609","bro_timestamp":"1530978698.168044","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702590","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978702590","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978702597","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"ChNCWL3i4gNIYPkoDe","trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49194 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/?60dbe33b908e0086292196ef001816bc tags:[] uid:ChNCWL3i4gNIYPkoDe trans_depth:1 host:62.75.195.236 status_msg:OK id.orig_h:192.168.138.158 response_body_len:0 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .
NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.168044 id.resp_h:62.75.195.236","ip_dst_addr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978702612","host":"62.75.195.236","enrichmentjoinbolt.joiner.ts":"1530978702601","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702605","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","timestamp":1530978698168,"method":"GET","request_body_len":0,"uri":"\/?60dbe33b908e0086292196ef001816bc","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978702597","threatintelsplitterbolt.splitter.end.ts":"1530978702605","adapter.threatinteladapter.begin.ts":"1530978702609","ip_src_port":49194,"enrichments.geo.ip_dst_addr.locati
on_point":"48.5839,7.7455","status_msg":"OK","guid":"5bd73342-6081-4de8-af0d-b68efab3bf95","response_body_len":0}
++{"adapter.threatinteladapter.end.ts":"1530978702609","bro_timestamp":"1530978698.840044","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702590","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702590","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702597","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"CX5zuR35fzQMB5VJmd","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49205 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/11iQmfg tags:[] uid:CX5zuR35fzQMB5VJmd resp_mime_types:[\"text\\\/html\"] trans_depth:1 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:192.168.138.158 response_body_len:3289 user_agent:Mozilla\/4.0 (com
patible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.840044 id.resp_h:95.163.121.204 resp_fuids:[\"FjwIxc3D3tcVPcqmGc\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702612","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702601","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702605","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FjwIxc3D3tcVPcqmGc"],"timestamp":1530978698840,"method":"GET","request_body_len":0,"uri":"\/11iQmfg","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978702597","threatintelsplitterbolt.splitter.en
d.ts":"1530978702605","adapter.threatinteladapter.begin.ts":"1530978702609","ip_src_port":49205,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"f9dbf04b-bcc8-48a9-b858-8ca45b6f8274","response_body_len":3289}
++{"adapter.threatinteladapter.end.ts":"1530978702609","bro_timestamp":"1530978698.949395","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702590","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978702590","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978702598","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"C8Ljn32fwV1v4G45R8","trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49188 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/aa25f5fe2875e3d0a244e6969e589cc4 tags:[] uid:C8Ljn32fwV1v4G45R8 trans_depth:1 host:62.75.195.236 status_msg:OK id.orig_h:192.168.138.158 response_body_len:861 ts:1530978698.949395 id.resp_h:62.75.195.236 resp_fuids:[\"FfQLue1qc3s7ZfGzH5\"]","ip_dst_a
ddr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978702612","host":"62.75.195.236","enrichmentjoinbolt.joiner.ts":"1530978702601","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702605","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","resp_fuids":["FfQLue1qc3s7ZfGzH5"],"timestamp":1530978698949,"method":"GET","request_body_len":0,"uri":"\/aa25f5fe2875e3d0a244e6969e589cc4","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978702598","threatintelsplitterbolt.splitter.end.ts":"1530978702605","adapter.threatinteladapter.begin.ts":"1530978702609","ip_src_port":49188,"enrichments.geo.ip_dst_addr.location_point":"48.5839,7.7455","status_msg":"OK","guid":"e8cff10e-d3ab-4578-8aeb-f94c614b5bd6","response_body_len":861}
++{"adapter.threatinteladapter.end.ts":"1530978702609","qclass_name":"qclass-32769","bro_timestamp":"1530978698.075525","qtype_name":"PTR","ip_dst_port":5353,"enrichmentsplitterbolt.splitter.end.ts":"1530978702590","qtype":12,"rejected":false,"enrichmentsplitterbolt.splitter.begin.ts":"1530978702590","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","trans_id":0,"adapter.geoadapter.begin.ts":"1530978702598","uid":"C8DgGj1pj2jXyhi9g1","protocol":"dns","original_string":"DNS | AA:false qclass_name:qclass-32769 id.orig_p:5353 qtype_name:PTR qtype:12 rejected:false id.resp_p:5353 query:_googlecast._tcp.local trans_id:0 TC:false RA:false uid:C8DgGj1pj2jXyhi9g1 RD:false proto:udp id.orig_h:192.168.66.1 Z:0 qclass:32769 ts:1530978698.075525 id.resp_h:224.0.0.251","ip_dst_addr":"224.0.0.251","threatinteljoinbolt.joiner.ts":"1530978702612","enrichmentjoinbolt.joiner.ts":"1530978702602","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts
":"1530978702605","Z":0,"ip_src_addr":"192.168.66.1","qclass":32769,"timestamp":1530978698075,"AA":false,"query":"_googlecast._tcp.local","TC":false,"RA":false,"source.type":"bro","adapter.geoadapter.end.ts":"1530978702598","RD":false,"threatintelsplitterbolt.splitter.end.ts":"1530978702606","adapter.threatinteladapter.begin.ts":"1530978702609","ip_src_port":5353,"proto":"udp","guid":"f88c60ba-4062-411f-ae82-c9a86e0a0d1b"}
++{"adapter.threatinteladapter.end.ts":"1530978702609","bro_timestamp":"1530978698.312623","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702590","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978702590","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978702598","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"CCTaln3ggV4dOqGETi","trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49194 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/?60dbe33b908e0086292196ef001816bc tags:[] uid:CCTaln3ggV4dOqGETi trans_depth:1 host:62.75.195.236 status_msg:OK id.orig_h:192.168.138.158 response_body_len:0 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .
NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.312623 id.resp_h:62.75.195.236","ip_dst_addr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978702612","host":"62.75.195.236","enrichmentjoinbolt.joiner.ts":"1530978702602","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702606","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","timestamp":1530978698312,"method":"GET","request_body_len":0,"uri":"\/?60dbe33b908e0086292196ef001816bc","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978702598","threatintelsplitterbolt.splitter.end.ts":"1530978702606","adapter.threatinteladapter.begin.ts":"1530978702609","ip_src_port":49194,"enrichments.geo.ip_dst_addr.locati
on_point":"48.5839,7.7455","status_msg":"OK","guid":"c0049477-bdc9-42ab-88fe-c088a7d9e76d","response_body_len":0}
++{"adapter.threatinteladapter.end.ts":"1530978702611","bro_timestamp":"1530978698.907146","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702591","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702591","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702598","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"Cnd9EM1uTP3PbJ0BS","resp_mime_types":["image\/png"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49209 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/flags\/de.png tags:[] uid:Cnd9EM1uTP3PbJ0BS referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:1 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h
:192.168.138.158 response_body_len:534 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.907146 id.resp_h:95.163.121.204 resp_fuids:[\"Fck5px3MJLpCrDeCZ3\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702613","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702602","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702607","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["Fck5px3MJLpCrDeCZ3"],"timestamp":1530978698907,"method":"GET","request_body_len":0,"uri":"\/img\/flags\/de.png","tags":[],"source.type":"bro","a
dapter.geoadapter.end.ts":"1530978702598","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978702607","adapter.threatinteladapter.begin.ts":"1530978702611","ip_src_port":49209,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"358e4eca-0f08-4c10-a307-881009c223b0","response_body_len":534}
++{"adapter.threatinteladapter.end.ts":"1530978702611","bro_timestamp":"1530978698.884865","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702591","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702591","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702598","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"CJY1nx4uy46hVP4kmg","resp_mime_types":["text\/plain"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49206 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/style.css tags:[] uid:CJY1nx4uy46hVP4kmg referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"text\\\/plain\"] trans_depth:1 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h
:192.168.138.158 response_body_len:4492 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.884865 id.resp_h:95.163.121.204 resp_fuids:[\"FindPO2TsX283BvQw3\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702613","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702602","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702607","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FindPO2TsX283BvQw3"],"timestamp":1530978698884,"method":"GET","request_body_len":0,"uri":"\/img\/style.css","tags":[],"source.type":"bro","adap
ter.geoadapter.end.ts":"1530978702598","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978702607","adapter.threatinteladapter.begin.ts":"1530978702611","ip_src_port":49206,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"94df9cde-8877-43bf-97a6-d2e0bbc840c4","response_body_len":4492}
++{"adapter.threatinteladapter.end.ts":"1530978702611","bro_timestamp":"1530978698.521985","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702591","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702591","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702598","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"C1qlzE2SalKbpWSJGi","resp_mime_types":["image\/png"],"trans_depth":3,"protocol":"http","original_string":"HTTP | id.orig_p:49210 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/button_pay.png tags:[] uid:C1qlzE2SalKbpWSJGi referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:3 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.ori
g_h:192.168.138.158 response_body_len:727 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.521985 id.resp_h:95.163.121.204 resp_fuids:[\"Fd2ecB4nK7EKV7lLA1\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702613","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702602","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702607","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["Fd2ecB4nK7EKV7lLA1"],"timestamp":1530978698521,"method":"GET","request_body_len":0,"uri":"\/img\/button_pay.png","tags":[],"source.type":"bro
","adapter.geoadapter.end.ts":"1530978702598","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978702607","adapter.threatinteladapter.begin.ts":"1530978702611","ip_src_port":49210,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"024a7ece-fce2-4ec2-86ee-e5e7d0dc2a5d","response_body_len":727}
++{"adapter.threatinteladapter.end.ts":"1530978702611","bro_timestamp":"1530978698.077529","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702591","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702591","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702598","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"C5UfKV32U65H7ojqJd","resp_mime_types":["image\/png"],"trans_depth":4,"protocol":"http","original_string":"HTTP | id.orig_p:49205 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/bitcoin.png tags:[] uid:C5UfKV32U65H7ojqJd referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:4 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h
:192.168.138.158 response_body_len:5523 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.077529 id.resp_h:95.163.121.204 resp_fuids:[\"Fy6w2R347d11rin2hg\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702613","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702602","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702607","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["Fy6w2R347d11rin2hg"],"timestamp":1530978698077,"method":"GET","request_body_len":0,"uri":"\/img\/bitcoin.png","tags":[],"source.type":"bro","ad
apter.geoadapter.end.ts":"1530978702598","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978702607","adapter.threatinteladapter.begin.ts":"1530978702611","ip_src_port":49205,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"626f9f8b-2af5-4c9f-a36a-3cfedea5614e","response_body_len":5523}
++{"adapter.threatinteladapter.end.ts":"1530978702611","bro_timestamp":"1530978698.241724","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702591","enrichments.geo.ip_dst_addr.city":"Los Angeles","enrichments.geo.ip_dst_addr.latitude":"34.0494","enrichmentsplitterbolt.splitter.begin.ts":"1530978702591","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5368361","adapter.geoadapter.begin.ts":"1530978702598","enrichments.geo.ip_dst_addr.postalCode":"90014","uid":"CJNiGM3zcyXHHORzFb","resp_mime_types":["text\/plain"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49198 status_code:200 method:POST request_body_len:134 id.resp_p:80 orig_mime_types:[\"text\\\/plain\"] uri:\/wp-content\/themes\/grizzly\/img5.php?c=cdcnw7cfz43rmtg tags:[] uid:CJNiGM3zcyXHHORzFb resp_mime_types:[\"text\\\/plain\"] trans_depth:1 orig_fuids:[\"FJWjcF3Z0qYg56Pw65\"] hos
t:comarksecurity.com status_msg:OK id.orig_h:192.168.138.158 response_body_len:14 user_agent:Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.241724 id.resp_h:72.34.49.86 resp_fuids:[\"FrVBRXxij8xG1u239\"]","ip_dst_addr":"72.34.49.86","threatinteljoinbolt.joiner.ts":"1530978702614","enrichments.geo.ip_dst_addr.dmaCode":"803","host":"comarksecurity.com","enrichmentjoinbolt.joiner.ts":"1530978702603","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702608","enrichments.geo.ip_dst_addr.longitude":"-118.2641","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FrVBRXxij8xG1u239"],"timestamp":1530978698241,"method":"POST","request_body
_len":134,"orig_mime_types":["text\/plain"],"uri":"\/wp-content\/themes\/grizzly\/img5.php?c=cdcnw7cfz43rmtg","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978702598","threatintelsplitterbolt.splitter.end.ts":"1530978702608","adapter.threatinteladapter.begin.ts":"1530978702611","orig_fuids":["FJWjcF3Z0qYg56Pw65"],"ip_src_port":49198,"enrichments.geo.ip_dst_addr.location_point":"34.0494,-118.2641","status_msg":"OK","guid":"40b1b0b6-a51c-41a2-9d97-ef26badb79fa","response_body_len":14}
++{"adapter.threatinteladapter.end.ts":"1530978710497","bro_timestamp":"1530978704.958145","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978710473","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978710473","adapter.hostfromjsonlistadapter.end.ts":"1530978710476","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978710476","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"C7KeXZ1jvzj9qkSqt7","resp_mime_types":["application\/x-shockwave-flash"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49185 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:C7KeXZ1jvzj9qkSqt7 referrer:http:\/\/va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in\/?285a4d4e4e5a4d4d4649584c5d43064b4745 resp_mime_types:[\"application
\\\/x-shockwave-flash\"] trans_depth:1 host:ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in status_msg:OK id.orig_h:192.168.138.158 response_body_len:8973 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978704.958145 id.resp_h:62.75.195.236 resp_fuids:[\"Ft6fqj1vE6fmJBYPx6\"]","ip_dst_addr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978710501","host":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","enrichmentjoinbolt.joiner.ts":"1530978710487","adapter.hostfromjsonlistadapter.begin.ts":"1530978710475","threatintelsplitterbolt.splitter.begin.ts":"1530978710493","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.3
0729; Media Center PC 6.0)","resp_fuids":["Ft6fqj1vE6fmJBYPx6"],"timestamp":1530978704958,"method":"GET","request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978710476","referrer":"http:\/\/va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in\/?285a4d4e4e5a4d4d4649584c5d43064b4745","threatintelsplitterbolt.splitter.end.ts":"1530978710493","adapter.threatinteladapter.begin.ts":"1530978710497","ip_src_port":49185,"enrichments.geo.ip_dst_addr.location_point":"48.5839,7.7455","status_msg":"OK","guid":"f1d5ef09-d2e5-4cdd-a26b-fc23df82c385","response_body_len":8973}
++{"adapter.threatinteladapter.end.ts":"1530978710497","bro_timestamp":"1530978704.608287","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978710477","enrichmentsplitterbolt.splitter.begin.ts":"1530978710477","adapter.hostfromjsonlistadapter.end.ts":"1530978710486","adapter.geoadapter.begin.ts":"1530978710486","uid":"CUrRne3iLIxXavQtci","trans_depth":40,"protocol":"http","original_string":"HTTP | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 uri:\/api\/v1\/persist\/wizard-data?_=1484168498643 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/ trans_depth:40 host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36 ts:1530978704.608287 id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1530978710501","host":"node1","enrichmentjoinbolt.joiner.ts":"1530978710491","adapter.host
fromjsonlistadapter.begin.ts":"1530978710486","threatintelsplitterbolt.splitter.begin.ts":"1530978710493","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36","timestamp":1530978704608,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/persist\/wizard-data?_=1484168498643","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978710486","referrer":"http:\/\/node1:8080\/","threatintelsplitterbolt.splitter.end.ts":"1530978710493","adapter.threatinteladapter.begin.ts":"1530978710497","ip_src_port":50451,"guid":"438b5c9d-522a-4611-9d70-c3723645611b","response_body_len":0}
++{"adapter.threatinteladapter.end.ts":"1530978710499","bro_timestamp":"1530978704.063932","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978710477","enrichmentsplitterbolt.splitter.begin.ts":"1530978710477","adapter.hostfromjsonlistadapter.end.ts":"1530978710489","adapter.geoadapter.begin.ts":"1530978710490","uid":"CUrRne3iLIxXavQtci","trans_depth":178,"protocol":"http","original_string":"HTTP | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 uri:\/api\/v1\/clusters\/metron_cluster\/components\/?ServiceComponentInfo\/component_name=APP_TIMELINE_SERVER|ServiceComponentInfo\/category=MASTER&fields=ServiceComponentInfo\/service_name,host_components\/HostRoles\/display_name,host_components\/HostRoles\/host_name,host_components\/HostRoles\/state,host_components\/HostRoles\/maintenance_state,host_components\/HostRoles\/stale_configs,host_components\/HostRoles\/ha_state,host_components\/HostRoles\/desired_admin_state,,host_components\/metrics\/jvm\/memHeapUse
dM,host_components\/metrics\/jvm\/HeapMemoryMax,host_components\/metrics\/jvm\/HeapMemoryUsed,host_components\/metrics\/jvm\/memHeapCommittedM,host_components\/metrics\/mapred\/jobtracker\/trackers_decommissioned,host_components\/metrics\/cpu\/cpu_wio,host_components\/metrics\/rpc\/client\/RpcQueueTime_avg_time,host_components\/metrics\/dfs\/FSNamesystem\/*,host_components\/metrics\/dfs\/namenode\/Version,host_components\/metrics\/dfs\/namenode\/LiveNodes,host_components\/metrics\/dfs\/namenode\/DeadNodes,host_components\/metrics\/dfs\/namenode\/DecomNodes,host_components\/metrics\/dfs\/namenode\/TotalFiles,host_components\/metrics\/dfs\/namenode\/UpgradeFinalized,host_components\/metrics\/dfs\/namenode\/Safemode,host_components\/metrics\/runtime\/StartTime,host_components\/metrics\/hbase\/master\/IsActiveMaster,host_components\/metrics\/hbase\/master\/MasterStartTime,host_components\/metrics\/hbase\/master\/MasterActiveTime,host_components\/metrics\/hbase\/master\/AverageLoad,host_
components\/metrics\/master\/AssignmentManger\/ritCount,metrics\/api\/v1\/cluster\/summary,metrics\/api\/v1\/topology\/summary,metrics\/api\/v1\/nimbus\/summary,host_components\/metrics\/yarn\/Queue,host_components\/metrics\/yarn\/ClusterMetrics\/NumActiveNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumLostNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumUnhealthyNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumRebootedNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumDecommissionedNMs&minimal_response=true&_=1484169119448 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/ trans_depth:178 host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36 ts:1530978704.063932 id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1530978710502","host":"node1","enrichmentjoinbolt.joiner.ts":"153
0978710494","adapter.hostfromjsonlistadapter.begin.ts":"1530978710489","threatintelsplitterbolt.splitter.begin.ts":"1530978710497","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36","timestamp":1530978704063,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/clusters\/metron_cluster\/components\/?ServiceComponentInfo\/component_name=APP_TIMELINE_SERVER|ServiceComponentInfo\/category=MASTER&fields=ServiceComponentInfo\/service_name,host_components\/HostRoles\/display_name,host_components\/HostRoles\/host_name,host_components\/HostRoles\/state,host_components\/HostRoles\/maintenance_state,host_components\/HostRoles\/stale_configs,host_components\/HostRoles\/ha_state,host_components\/HostRoles\/desired_admin_state,,host_components\/metrics\/jvm\/memHeapUsedM,host_components\/metrics\/jvm\/HeapMemoryMax,host_components\/metrics\/jvm\/HeapMemoryUsed,host_components\/me
trics\/jvm\/memHeapCommittedM,host_components\/metrics\/mapred\/jobtracker\/trackers_decommissioned,host_components\/metrics\/cpu\/cpu_wio,host_components\/metrics\/rpc\/client\/RpcQueueTime_avg_time,host_components\/metrics\/dfs\/FSNamesystem\/*,host_components\/metrics\/dfs\/namenode\/Version,host_components\/metrics\/dfs\/namenode\/LiveNodes,host_components\/metrics\/dfs\/namenode\/DeadNodes,host_components\/metrics\/dfs\/namenode\/DecomNodes,host_components\/metrics\/dfs\/namenode\/TotalFiles,host_components\/metrics\/dfs\/namenode\/UpgradeFinalized,host_components\/metrics\/dfs\/namenode\/Safemode,host_components\/metrics\/runtime\/StartTime,host_components\/metrics\/hbase\/master\/IsActiveMaster,host_components\/metrics\/hbase\/master\/MasterStartTime,host_components\/metrics\/hbase\/master\/MasterActiveTime,host_components\/metrics\/hbase\/master\/AverageLoad,host_components\/metrics\/master\/AssignmentManger\/ritCount,metrics\/api\/v1\/cluster\/summary,metrics\/api\/v1\/topo
logy\/summary,metrics\/api\/v1\/nimbus\/summary,host_components\/metrics\/yarn\/Queue,host_components\/metrics\/yarn\/ClusterMetrics\/NumActiveNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumLostNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumUnhealthyNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumRebootedNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumDecommissionedNMs&minimal_response=true&_=1484169119448","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978710490","referrer":"http:\/\/node1:8080\/","threatintelsplitterbolt.splitter.end.ts":"1530978710497","adapter.threatinteladapter.begin.ts":"1530978710499","ip_src_port":50451,"guid":"03546910-68c3-4aa4-90a3-6983bc23324e","response_body_len":0}
++{"adapter.threatinteladapter.end.ts":"1530978710499","bro_timestamp":"1530978704.137918","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978710478","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978710478","adapter.hostfromjsonlistadapter.end.ts":"1530978710489","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978710490","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"Cx8Ucg1r67RywyWab1","resp_mime_types":["image\/png"],"trans_depth":2,"protocol":"http","original_string":"HTTP | id.orig_p:49205 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/flags\/us.png tags:[] uid:Cx8Ucg1r67RywyWab1 referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:2 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig
_h:192.168.138.158 response_body_len:825 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978704.137918 id.resp_h:95.163.121.204 resp_fuids:[\"FCr63p4t8M7SUAumi3\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978710502","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978710494","adapter.hostfromjsonlistadapter.begin.ts":"1530978710489","threatintelsplitterbolt.splitter.begin.ts":"1530978710497","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FCr63p4t8M7SUAumi3"],"timestamp":1530978704137,"method":"GET","request_body_len":0,"uri":"\/img\/flags\/us.png","tags":[],"source.type":"bro",
"adapter.geoadapter.end.ts":"1530978710490","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978710497","adapter.threatinteladapter.begin.ts":"1530978710499","ip_src_port":49205,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"5ea15274-bf38-423e-9c3e-6fb0f3bf0270","response_body_len":825}
++{"adapter.threatinteladapter.end.ts":"1530978710499","bro_timestamp":"1530978704.973595","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978710478","enrichmentsplitterbolt.splitter.begin.ts":"1530978710478","adapter.hostfromjsonlistadapter.end.ts":"1530978710489","adapter.geoadapter.begin.ts":"1530978710490","uid":"CUrRne3iLIxXavQtci","trans_depth":251,"protocol":"http","original_string":"HTTP | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 uri:\/api\/v1\/clusters\/metron_cluster?fields=Clusters\/desired_configs\/cluster-env&_=1484169429016 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/ trans_depth:251 host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36 ts:1530978704.973595 id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1530978710502","host":"node1","enrichm
entjoinbolt.joiner.ts":"1530978710494","adapter.hostfromjsonlistadapter.begin.ts":"1530978710489","threatintelsplitterbolt.splitter.begin.ts":"1530978710497","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36","timestamp":1530978704973,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/clusters\/metron_cluster?fields=Clusters\/desired_configs\/cluster-env&_=1484169429016","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978710490","referrer":"http:\/\/node1:8080\/","threatintelsplitterbolt.splitter.end.ts":"1530978710497","adapter.threatinteladapter.begin.ts":"1530978710499","ip_src_port":50451,"guid":"273e3d59-b616-424e-8c30-add81bd671b9","response_body_len":0}
++{"adapter.threatinteladapter.end.ts":"1530978710499","bro_timestamp":"1530978704.973117","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978710478","enrichmentsplitterbolt.splitter.begin.ts":"1530978710478","adapter.hostfromjsonlistadapter.end.ts":"1530978710489","adapter.geoadapter.begin.ts":"1530978710490","uid":"CUrRne3iLIxXavQtci","trans_depth":247,"protocol":"http","original_string":"HTTP | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 uri:\/api\/v1\/clusters?fields=Clusters\/provisioning_state&_=1484169420015 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/ trans_depth:247 host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36 ts:1530978704.973117 id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1530978710502","host":"node1","enrichmentjoinbolt.joiner.ts":"15
30978710494","adapter.hostfromjsonlistadapter.begin.ts":"1530978710489","threatintelsplitterbolt.splitter.begin.ts":"1530978710497","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36","timestamp":1530978704973,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/clusters?fields=Clusters\/provisioning_state&_=1484169420015","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978710490","referrer":"http:\/\/node1:8080\/","threatintelsplitterbolt.splitter.end.ts":"1530978710497","adapter.threatinteladapter.begin.ts":"1530978710499","ip_src_port":50451,"guid":"9ba63d37-9e6b-4ba8-8504-cc418d9ce8aa","response_body_len":0}
++{"adapter.threatinteladapter.end.ts":"1530978710499","qclass_name":"C_INTERNET","bro_timestamp":"1530978704.094553","qtype_name":"PTR","ip_dst_port":5353,"enrichmentsplitterbolt.splitter.end.ts":"1530978710479","qtype":12,"rejected":false,"enrichmentsplitterbolt.splitter.begin.ts":"1530978710479","adapter.hostfromjsonlistadapter.end.ts":"1530978710489","trans_id":0,"adapter.geoadapter.begin.ts":"1530978710490","uid":"C03Lir2lgO0AxyDctk","protocol":"dns","original_string":"DNS | AA:false qclass_name:C_INTERNET id.orig_p:5353 qtype_name:PTR qtype:12 rejected:false id.resp_p:5353 query:_googlecast._tcp.local trans_id:0 TC:false RA:false uid:C03Lir2lgO0AxyDctk RD:false proto:udp id.orig_h:192.168.66.1 Z:0 qclass:1 ts:1530978704.094553 id.resp_h:224.0.0.251","ip_dst_addr":"224.0.0.251","threatinteljoinbolt.joiner.ts":"1530978710502","enrichmentjoinbolt.joiner.ts":"1530978710494","adapter.hostfromjsonlistadapter.begin.ts":"1530978710489","threatintelsplitterbolt.splitter.begin.ts":"15309
78710497","Z":0,"ip_src_addr":"192.168.66.1","qclass":1,"timestamp":1530978704094,"AA":false,"query":"_googlecast._tcp.local","TC":false,"RA":false,"source.type":"bro","adapter.geoadapter.end.ts":"1530978710490","RD":false,"threatintelsplitterbolt.splitter.end.ts":"1530978710497","adapter.threatinteladapter.begin.ts":"1530978710499","ip_src_port":5353,"proto":"udp","guid":"b5849fa0-3b1f-44a8-8b89-0eb3e823ba6f"}
++{"adapter.threatinteladapter.end.ts":"1530978710499","bro_timestamp":"1530978704.896579","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978710479","enrichmentsplitterbolt.splitter.begin.ts":"1530978710479","adapter.hostfromjsonlistadapter.end.ts":"1530978710489","adapter.geoadapter.begin.ts":"1530978710490","uid":"CUrRne3iLIxXavQtci","trans_depth":132,"protocol":"http","original_string":"HTTP | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 uri:\/api\/v1\/clusters\/metron_cluster\/components\/?fields=ServiceComponentInfo\/service_name,ServiceComponentInfo\/category,ServiceComponentInfo\/installed_count,ServiceComponentInfo\/started_count,ServiceComponentInfo\/init_count,ServiceComponentInfo\/install_failed_count,ServiceComponentInfo\/unknown_count,ServiceComponentInfo\/total_count,ServiceComponentInfo\/display_name,host_components\/HostRoles\/host_name&minimal_response=true&_=1484168884281 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/
trans_depth:132 host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36 ts:1530978704.896579 id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1530978710502","host":"node1","enrichmentjoinbolt.joiner.ts":"1530978710495","adapter.hostfromjsonlistadapter.begin.ts":"1530978710489","threatintelsplitterbolt.splitter.begin.ts":"1530978710497","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36","timestamp":1530978704896,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/clusters\/metron_cluster\/components\/?fields=ServiceComponentInfo\/service_name,ServiceComponentInfo\/category,ServiceComponentInfo\/installed_count,ServiceComponentInfo\/started_count,ServiceComponentInfo\/init_count,ServiceComponen
tInfo\/install_failed_count,ServiceComponentInfo\/unknown_count,ServiceComponentInfo\/total_count,ServiceComponentInfo\/display_name,host_components\/HostRoles\/host_name&minimal_response=true&_=1484168884281","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978710490","referrer":"http:\/\/node1:8080\/","threatintelsplitterbolt.splitter.end.ts":"1530978710497","adapter.threatinteladapter.begin.ts":"1530978710499","ip_src_port":50451,"guid":"bc257399-d461-4dd2-b6ac-d18c26af2dd2","response_body_len":0}
++{"adapter.threatinteladapter.end.ts":"1530978710499","bro_timestamp":"1530978704.015832","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978710479","enrichments.geo.ip_dst_addr.city":"Los Angeles","enrichments.geo.ip_dst_addr.latitude":"34.0494","enrichmentsplitterbolt.splitter.begin.ts":"1530978710479","adapter.hostfromjsonlistadapter.end.ts":"1530978710489","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5368361","adapter.geoadapter.begin.ts":"1530978710490","enrichments.geo.ip_dst_addr.postalCode":"90014","uid":"CpBTZB1XlDvW4TC9o4","resp_mime_types":["image\/png"],"trans_depth":1,"protocol":"ht
<TRUNCATED>
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-deployment/packaging/docker/deb-docker/pom.xml
----------------------------------------------------------------------
[50/51] [abbrv] metron git commit: METRON-1826 Update librdkafka and
devtoolset (JonZeolla via jonzeolla) closes apache/metron#1238
Posted by ot...@apache.org.
METRON-1826 Update librdkafka and devtoolset (JonZeolla via jonzeolla) closes apache/metron#1238
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0ba18026
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0ba18026
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0ba18026
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 0ba18026222f004316b96633780509c71934121e
Parents: 739782a
Author: JonZeolla <ze...@gmail.com>
Authored: Mon Oct 22 19:48:56 2018 -0400
Committer: jonzeolla <jo...@apache.org>
Committed: Mon Oct 22 19:48:56 2018 -0400
----------------------------------------------------------------------
.../ansible/roles/bro/tasks/bro.yml | 4 +--
.../ansible/roles/bro/tasks/dependencies.yml | 4 +--
.../roles/bro/tasks/metron-bro-plugin-kafka.yml | 4 +--
.../ansible/roles/librdkafka/defaults/main.yml | 4 +--
metron-deployment/development/centos6/README.md | 26 ++++++++++++++++++--
.../development/ubuntu14/README.md | 26 ++++++++++++++++++--
6 files changed, 56 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/ansible/roles/bro/tasks/bro.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/bro/tasks/bro.yml b/metron-deployment/ansible/roles/bro/tasks/bro.yml
index 222ef0e..9ba3ffa 100644
--- a/metron-deployment/ansible/roles/bro/tasks/bro.yml
+++ b/metron-deployment/ansible/roles/bro/tasks/bro.yml
@@ -30,8 +30,8 @@
- name: Compile and Install bro
shell: "{{ item }}"
environment:
- CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
- CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
+ CXX: /opt/rh/devtoolset-6/root/usr/bin/g++
+ CC: /opt/rh/devtoolset-6/root/usr/bin/gcc
args:
chdir: "/tmp/bro-{{ bro_version }}"
creates: "{{ bro_home }}/bin/bro"
http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/bro/tasks/dependencies.yml b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
index 3cd3bae..f8b1e28 100644
--- a/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
+++ b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
@@ -38,8 +38,8 @@
- perl
- crontabs
- net-tools
- - devtoolset-4-gcc
- - devtoolset-4-gcc-c++
+ - devtoolset-6-gcc
+ - devtoolset-6-gcc-c++
- python27
- rh-git29
register: result
http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
index 7043387..ec7af97 100644
--- a/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
+++ b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
@@ -30,8 +30,8 @@
environment:
PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}"
LD_LIBRARY_PATH: "{{ httpd24_lib }}:{{ python27_lib }}"
- CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
- CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
+ CXX: /opt/rh/devtoolset-6/root/usr/bin/g++
+ CC: /opt/rh/devtoolset-6/root/usr/bin/gcc
command: "{{ python27_bin }}/bro-pkg install apache/metron-bro-plugin-kafka --force --version {{ metron_bro_plugin_kafka_version }}"
- name: Configure bro-kafka plugin
http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
index 063c22f..6e25cb3 100644
--- a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
+++ b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
@@ -15,6 +15,6 @@
# limitations under the License.
#
---
-librdkafka_version: 0.9.4
-librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz
+librdkafka_version: 0.11.5
+librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz
librdkafka_home: /usr/local
http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/development/centos6/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/README.md b/metron-deployment/development/centos6/README.md
index e873452..99ec967 100644
--- a/metron-deployment/development/centos6/README.md
+++ b/metron-deployment/development/centos6/README.md
@@ -97,8 +97,30 @@ In addition to re-running the entire provisioning play book, you may now re-run
vagrant --ansible-tags="sensor-stubs" provision
```
-Tags are listed in the playbooks, some frequently used tags:
+Tags are listed in the playbooks. Here are some frequently used tags:
+ `hdp-install` - Install HDP
+ `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services)
-+ `sensors` - Deploy and start the sensors.
++ `sensors` - Deploy the sensors (see [Sensors](#sensors) for more details regarding this tag)
+ `sensor-stubs` - Deploy and start the sensor stubs.
+
+#### Sensors
+
+By default, the Metron development environment uses sensor stubs to mimic the behavior of the full sensors. This is done because the full sensors take a significant amount of time and CPU to build, install, and run.
+
+From time to time you may want to install the full sensors for testing (see the specifics of what that means [here](../../ansible/playbooks/sensor_install.yml)). This can be done by running the following command:
+
+```
+vagrant --ansible-skip-tags="sensor-stubs" up
+```
+
+This will skip only the `sensor-stubs` tag, allowing the ansible roles with the `sensors` tag to be run. This provisions the full sensors in a 'testing mode' so that they are more active, and thus more useful for testing (more details on that [here](../../ansible/roles/sensor-test-mode/)). **However**, when vagrant completes the sensors will NOT be running. In order to start the sensors and simulate traffic through them (which will create a fair amount of load on your test system), complete the below steps:
+
+```
+vagrant ssh
+sudo su -
+service pcap-replay restart
+service yaf restart
+service snortd restart
+service snort-producer restart
+```
+
http://git-wip-us.apache.org/repos/asf/metron/blob/0ba18026/metron-deployment/development/ubuntu14/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/README.md b/metron-deployment/development/ubuntu14/README.md
index 494937e..53bea67 100644
--- a/metron-deployment/development/ubuntu14/README.md
+++ b/metron-deployment/development/ubuntu14/README.md
@@ -98,8 +98,30 @@ In addition to re-running the entire provisioning play book, you may now re-run
vagrant --ansible-tags="sensor-stubs" provision
```
-Tags are listed in the playbooks, some frequently used tags:
+Tags are listed in the playbooks. Here are some frequently used tags:
+ `hdp-install` - Install HDP
+ `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services)
-+ `sensors` - Deploy and start the sensors.
++ `sensors` - Deploy the sensors (see [Sensors](#sensors) for more details regarding this tag)
+ `sensor-stubs` - Deploy and start the sensor stubs.
+
+#### Sensors
+
+By default, the Metron development environment uses sensor stubs to mimic the behavior of the full sensors. This is done because the full sensors take a significant amount of time and CPU to build, install, and run.
+
+From time to time you may want to install the full sensors for testing (see the specifics of what that means [here](../../ansible/playbooks/sensor_install.yml)). This can be done by running the following command:
+
+```
+vagrant --ansible-skip-tags="sensor-stubs" up
+```
+
+This will skip only the `sensor-stubs` tag, allowing the ansible roles with the `sensors` tag to be run. This provisions the full sensors in a 'testing mode' so that they are more active, and thus more useful for testing (more details on that [here](../../ansible/roles/sensor-test-mode/)). **However**, when vagrant completes the sensors will NOT be running. In order to start the sensors and simulate traffic through them (which will create a fair amount of load on your test system), complete the below steps:
+
+```
+vagrant ssh
+sudo su -
+service pcap-replay restart
+service yaf restart
+service snortd restart
+service snort-producer restart
+```
+
[45/51] [abbrv] metron git commit: METRON-1816 Date format Stellar
function (merrimanr) closes apache/metron#1233
Posted by ot...@apache.org.
METRON-1816 Date format Stellar function (merrimanr) closes apache/metron#1233
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bf32fef6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bf32fef6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bf32fef6
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: bf32fef6a4a02c5b21ec1f6d97fbd83de78a996e
Parents: 28542ad
Author: merrimanr <me...@gmail.com>
Authored: Thu Oct 18 17:00:44 2018 -0500
Committer: rmerriman <me...@gmail.com>
Committed: Thu Oct 18 17:00:44 2018 -0500
----------------------------------------------------------------------
metron-stellar/stellar-common/README.md | 9 +++++
.../stellar/dsl/functions/DateFunctions.java | 42 ++++++++++++++++++++
.../dsl/functions/DateFunctionsTest.java | 38 ++++++++++++++++++
3 files changed, 89 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/bf32fef6/metron-stellar/stellar-common/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md
index 468f358..0f3bb6a 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -169,6 +169,7 @@ Where:
| [ `CHOP`](#chop) |
| [ `CHOMP`](#chomp) |
| [ `COUNT_MATCHES`](#count_matches) |
+| [ `DATE_FORMAT`](#date_format)
| [ `DAY_OF_MONTH`](#day_of_month) |
| [ `DAY_OF_WEEK`](#day_of_week) |
| [ `DAY_OF_YEAR`](#day_of_year) |
@@ -379,6 +380,14 @@ Where:
* substring/character - the substring or character to count, may be null.
* Returns: the number of non-overlapping occurrences, 0 if either CharSequence is null.
+### `DATE_FORMAT`
+ * Description: Takes an epoch timestamp and converts it to a date format.
+ * Input:
+ * format - DateTime format as a String.
+ * timestampField - Optional epoch time in Long format. Defaults to now.
+ * timezone - Optional timezone in String format.
+ * Returns: Formatted date.
+
### `DAY_OF_MONTH`
* Description: The numbered day within the month. The first day within the month has a value of 1.
* Input:
http://git-wip-us.apache.org/repos/asf/metron/blob/bf32fef6/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
index 212d6e9..17f5f8d 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
@@ -28,6 +28,7 @@ import org.apache.metron.stellar.common.utils.ConversionUtils;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
+import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.TimeZone;
@@ -109,6 +110,13 @@ public class DateFunctions {
return sdf.parse(date).getTime();
}
+ public static String getDateFormat(String format, Optional<Long> epochTime, Optional<String> timezone) {
+ Long time = epochTime.orElseGet(System::currentTimeMillis);
+ TimezonedFormat fmt = timezone.map(s -> new TimezonedFormat(format, s)).orElseGet(() -> new TimezonedFormat(format));
+ SimpleDateFormat sdf = formatCache.get(fmt).get();
+ return sdf.format(new Date(time));
+ }
+
/**
* Stellar Function: TO_EPOCH_TIMESTAMP
@@ -144,6 +152,40 @@ public class DateFunctions {
}
}
+ @Stellar( name="DATE_FORMAT",
+ description = "Takes an epoch timestamp and converts it to a date format.",
+ params = {"format - DateTime format as a String."
+ , "timestampField - Optional epoch time in Long format. Defaults to now."
+ , "timezone - Optional timezone in String format."},
+ returns="Formatted date."
+ )
+ public static class DateFormat extends BaseStellarFunction {
+
+ public Object apply(List<Object> objects) {
+ int size = objects.size();
+ Optional<Object> formatObj = Optional.ofNullable(objects.get(0));
+ Optional<Long> epochObj = Optional.empty();
+ Optional<String> tzObj = Optional.empty();
+ if (size > 1) {
+ if (size == 2) {
+ if (objects.get(1) == null) {
+ return null;
+ }
+ epochObj = objects.get(1) instanceof Long ? Optional.of((Long) objects.get(1)) : Optional.empty();
+ tzObj = objects.get(1) instanceof String ? Optional.of((String) objects.get(1)) : Optional.empty();
+ } else {
+ epochObj = Optional.ofNullable((Long) objects.get(1));
+ tzObj = Optional.ofNullable((String) objects.get(2));
+ }
+ }
+ if(formatObj.isPresent()) {
+ return getDateFormat(formatObj.get().toString(), epochObj, tzObj);
+ } else {
+ return null;
+ }
+ }
+ }
+
/**
* Gets the value from a list of arguments.
*
http://git-wip-us.apache.org/repos/asf/metron/blob/bf32fef6/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
index 1f1f4f4..959e7ea 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
@@ -31,6 +31,7 @@ import org.junit.Test;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Map;
+import java.util.TimeZone;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -225,4 +226,41 @@ public class DateFunctionsTest {
public void testDayOfYearNull() {
Object result = run("DAY_OF_YEAR(nada)");
}
+
+ @Test
+ public void testDateFormat() {
+ Object result = run("DATE_FORMAT('EEE MMM dd yyyy hh:mm:ss zzz', epoch, 'EST')");
+ assertEquals("Thu Aug 25 2016 08:27:10 EST", result);
+ }
+
+ @Test
+ public void testDateFormatDefault() {
+ Object result = run("DATE_FORMAT('EEE MMM dd yyyy hh:mm:ss zzzz')");
+ assertTrue(result.toString().endsWith(TimeZone.getDefault().getDisplayName(true, 1)));
+ }
+
+ @Test
+ public void testDateFormatNow() {
+ Object result = run("DATE_FORMAT('EEE MMM dd yyyy hh:mm:ss zzz', 'GMT')");
+ assertTrue(result.toString().endsWith("GMT"));
+ }
+
+ @Test
+ public void testDateFormatDefaultTimezone() {
+ Object result = run("DATE_FORMAT('EEE MMM dd yyyy hh:mm:ss zzzz', epoch)");
+ assertTrue(result.toString().endsWith(TimeZone.getDefault().getDisplayName(true, 1)));
+ }
+
+ /**
+ * If refer to variable that does not exist, expect ParseException.
+ */
+ @Test(expected = ParseException.class)
+ public void testDateFormatNull() {
+ Object result = run("DATE_FORMAT('EEE MMM dd yyyy hh:mm:ss zzz', nada, 'EST')");
+ }
+
+ @Test(expected = ParseException.class)
+ public void testDateFormatInvalid() {
+ Object result = run("DATE_FORMAT('INVALID DATE FORMAT', epoch, 'EST')");
+ }
}
[38/51] [abbrv] metron git commit: METRON-1799 Remove outdated bylaws
from site. (justinleet) closes apache/metron#1216
Posted by ot...@apache.org.
METRON-1799 Remove outdated bylaws from site. (justinleet) closes apache/metron#1216
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9b671005
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9b671005
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9b671005
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 9b6710053894d8a39880cb8157a0e603ed542cb7
Parents: f153375
Author: justinleet <ju...@gmail.com>
Authored: Thu Oct 11 08:41:23 2018 -0400
Committer: leet <le...@apache.org>
Committed: Thu Oct 11 08:41:23 2018 -0400
----------------------------------------------------------------------
site/develop/bylaws.md | 281 --------------------------------------------
site/develop/coding.md | 34 ------
site/develop/index.md | 61 ----------
3 files changed, 376 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/9b671005/site/develop/bylaws.md
----------------------------------------------------------------------
diff --git a/site/develop/bylaws.md b/site/develop/bylaws.md
deleted file mode 100644
index a8fc5fb..0000000
--- a/site/develop/bylaws.md
+++ /dev/null
@@ -1,281 +0,0 @@
----
-layout: page
-title: Apache Metron Bylaws
----
-
-## Introduction
-
-This document defines the bylaws under which the Apache Metron project
-operates. It defines the roles and responsibilities of the project,
-who may vote, how voting works, how conflicts are resolved, etc.
-
-Metron is a project of the Apache Software Foundation (ASF) and the foundation
-holds the trademark on the name "Metron" and copyright on the combined
-code base. The [Apache Foundation
-FAQ](http://www.apache.org/foundation/faq.html) and
-[How-It-Works](http://www.apache.org/foundation/how-it-works.html)
-explain the operation and background of the foundation.
-
-Apache has a [code of
-conduct](http://www.apache.org/foundation/policies/conduct.html) that
-it expects its members to follow. In particular:
-
-* Be **open and welcoming**. It is important that we grow and
- encourage the community of users and developers for our project.
-
-* Be **collaborative**. Working together on the open mailing lists and
- bug database to make decisions helps the project grow.
-
-* Be **respectful** of others. Everyone is volunteering their time and
- efforts to work on this project. Please be respectful of everyone
- and their views.
-
-Metron is typical of Apache projects in that it operates under a set of
-principles, known collectively as the "Apache Way". If you are new to
-Apache development, please refer to
-[this](http://www.apache.org/foundation/how-it-works.html) for more
-information on how Apache projects operate.
-
-## Roles and Responsibilities
-
-Apache projects define a set of roles with associated rights and
-responsibilities. These roles govern what tasks an individual may
-perform within the project. The roles are defined in the following
-sections.
-
-### Users
-
-The most important participants in the project are people who use our
-software. The majority of our developers start out as users and guide
-their development efforts from the user's perspective. Users
-contribute to the Apache projects by providing feedback to developers
-in the form of bug reports and feature suggestions. As well, users
-participate in the Apache community by helping other users on mailing
-lists and user support forums.
-
-### Contributors
-
-Contributors include all of the volunteers who donate time, code,
-documentation,
-or resources to the Metron Project. A contributor that makes sustained,
-welcome contributions to the project may be invited to become a
-committer, though the exact timing of such invitations depends on many
-factors.
-
-### Committers
-
-The project's committers are responsible for the project's technical
-management. Committers have the right to commit to the project's git
-repository. Committers may cast binding votes on any technical
-discussion.
-
-Committer access is by invitation only and must be approved by
-consensus approval of the active Project Management Committee (PMC)
-members.
-
-If a committer wishes to leave the project or does not contribute to
-the project in any form for six months, the PMC may make them emeritus.
-Emeritus committers lose their ability to commit code or cast binding
-votes. An emeritus committer may
-request reinstatement of commit access from the PMC. Such
-reinstatement is subject to consensus approval of active PMC members.
-
-All Apache committers are required to have a signed [Individual
-Contributor License
-Agreement](https://www.apache.org/licenses/icla.txt) (ICLA) on file
-with the Apache Software Foundation. There is a [Committer
-FAQ](http://www.apache.org/dev/committers.html) which provides more
-details on the requirements for Committers.
-
-A committer who makes a
-sustained contribution to the project may be invited to become a
-member of the PMC. The form of contribution
-is not limited to code. It can also include code review, helping out
-users on the mailing lists, documentation, testing, etc.
-
-### Release Manager
-
-A Release Manager (RM) is a committer who volunteers to produce a
-Release Candidate. The RM shall publish a Release Plan on the
-dev mailing list stating the branch from which they intend to
-make a Release Candidate.
-
-### Project Management Committee
-
-The Project Management Committee (PMC) for Apache Metron was created by
-the Apache Board in December 2015 when Metron moved out of Cisco's OpenSOC
-project and became an incubated project at Apache.
-The PMC is responsible to the board and
-the ASF for the management and oversight of the Apache Metron
-codebase. The responsibilities of the PMC include
-
- * Deciding what is distributed as products of the Apache Metron
- project. In particular all releases must be approved by the PMC.
-
- * Maintaining the project's shared resources, including the codebase
- repository, mailing lists, and websites.
-
- * Speaking on behalf of the project.
-
- * Resolving license disputes regarding products of the project
-
- * Nominating new PMC members and committers
-
- * Maintaining these bylaws and other guidelines of the project
-
-Membership of the PMC is by invitation only and must be approved by a
-consensus approval of active PMC members.
-
-A PMC member is considered
-emeritus by their own declaration or by not contributing in any form
-to the project for over six months. An emeritus member may request
-reinstatement to the PMC. Such reinstatement is subject to consensus
-approval of the active PMC members.
-
-The chair of the PMC is appointed by the ASF board. The chair is an
-office holder of the Apache Software Foundation (Vice President,
-Apache Metron) and has primary responsibility to the board for the
-management of the project within the scope of the Metron PMC. The
-chair reports to the board quarterly on developments within the Metron
-project.
-
-When the project desires a new PMC chair, the PMC votes to recommend a
-new chair using [Single Transferable
-Vote](http://wiki.apache.org/general/BoardVoting) voting. The decision
-must be ratified by the Apache board.
-
-## Decision Making
-
-Within the Metron project, different types of decisions require
-different forms of approval. For example, the previous section
-describes several decisions which require "consensus approval."
-This section defines how voting is performed, the types of
-approvals, and which types of decision require which type of approval.
-
-### Voting
-
-Decisions regarding the project are made by votes on the primary
-project development mailing list (dev@Metron.apache.org). Where
-necessary, PMC voting may take place on the private Metron PMC mailing
-list. Votes are clearly indicated by subject line starting with
-[VOTE]. Votes may contain multiple items for approval and these should
-be clearly separated. Voting is carried out by replying to the vote
-mail. Voting may take five flavors:
-
-* **+1** -- "Yes," "Agree," or "the action should be performed." In general,
- this vote also indicates a willingness on the behalf of the voter in
- "making it happen."
-
-* **+0** -- This vote indicates a willingness for the action under
- consideration to go ahead. The voter, however, will not be able to
- help.
-
-* **0** -- The voter is neutral on the topic under discussion.
-
-* **-0** -- This vote indicates that the voter does not, in general, agree
- with the proposed action but is not concerned enough to prevent the
- action going ahead.
-
-* **-1** -- This is a negative vote. On issues where consensus is required,
- this vote counts as a veto. All vetoes must contain an explanation
- of why the veto is appropriate. Vetoes with no explanation are
- void. It may also be appropriate for a -1 vote to include an
- alternative course of action.
-
-All participants in the Metron project are encouraged to show their
-agreement for or against a particular action by voting, regardless of
-whether their vote is binding. Nonbinding votes are useful for
-encouraging discussion and understanding the scope of opinions within
-the project.
-
-### Approvals
-
-These are the types of approvals that can be sought. Different actions
-require different types of approvals.
-
-* **Consensus Approval** -- Consensus approval requires 3 binding +1
- votes and no binding vetoes.
-
-* **Lazy Consensus** -- Lazy consensus requires at least one +1 vote and
- no -1 votes ('silence gives assent').
-
-* **Lazy Majority** -- A lazy majority vote requires 3 binding +1 votes
- and more binding +1 votes than -1 votes.
-
-* **Lazy 2/3 Majority** -- Lazy 2/3 majority votes requires at least 3
- votes and twice as many +1 votes as -1 votes.
-
-### Vetoes
-
-A valid, binding veto cannot be overruled. If a veto is cast, it must
-be accompanied by a valid reason explaining the reasons for the
-veto. The validity of a veto, if challenged, can be confirmed by
-anyone who has a binding vote. This does not necessarily signify
-agreement with the veto - merely that the veto is valid. If you
-disagree with a valid veto, you must lobby the person casting the veto
-to withdraw their veto. If a veto is not withdrawn, any action that
-has already been taken must be reversed in a timely manner.
-
-### Actions
-
-This section describes the various actions which are undertaken within
-the project, the corresponding approval required for that action and
-those who have binding votes over the action.
-
-#### Code Change
-
-A change made to a codebase of the project requires *lazy consensus*
-of active committers other than the author of the patch. The code can
-be committed after the first +1.
-
-#### Product Release
-
-To make a release, the release manager creates a release candidate and
-a vote requiring a *lazy majority* of the active PMC members is
-required. Once the vote passes, the release candidate becomes an
-official release.
-
-#### Adoption of New Codebase
-
-When the codebase for an existing, released product is to be replaced
-with an alternative codebase, it requires a *lazy 2/3 majority* of PMC
-members. This also covers the creation of new sub-projects and
-submodules within the project.
-
-#### New Committer
-
-When a new committer is proposed for the project, *consensus approval*
-of the active PMC members is required.
-
-#### New PMC Member
-
-To promote a committer to a PMC member requires *consensus approval*
-of active PMC members.
-
-If the vote passes, the Apache Board must be notified to make the change
-official.
-
-#### Committer Removal
-
-Removal of commit privileges requires a *lazy 2/3 majority* of active
-PMC members.
-
-#### PMC Member Removal
-
-Removing a PMC member requires a *lazy 2/3 majority* of active PMC
-members, excluding the member in question.
-
-If the vote passes, the Apache Board must be notified to make the change
-official.
-
-#### Modifying Bylaws
-
-Modifying this document requires a *lazy majority* of active PMC members.
-
-### Voting Timeframes
-
-Votes are open for a minimum period of 72 hours to allow all active
-voters time to consider the vote. For holiday weekends or conferences,
-consider using a longer vote window. Votes relating to code changes are
-not subject to a strict timetable but should be made as timely as
-possible.
http://git-wip-us.apache.org/repos/asf/metron/blob/9b671005/site/develop/coding.md
----------------------------------------------------------------------
diff --git a/site/develop/coding.md b/site/develop/coding.md
deleted file mode 100644
index 78c7e27..0000000
--- a/site/develop/coding.md
+++ /dev/null
@@ -1,34 +0,0 @@
----
-layout: page
-title: Coding Guidelines
----
-
-## General rules
-
-* All files must have an Apache copyright header at the top of the file.
-* Code should be removed rather than commented out.
-* All public functions should have javadoc comments.
-* Always use braces to surround branches.
-* try-finally should be avoided.
-
-## Formatting
-
-* All files must have an 80 character maximum line length.
-* Indentation should be 2 spaces.
-* Files should use spaces instead of tabs.
-* Wrapping lines
- * Break after a comma.
- * Break before an operator.
- * Prefer higher-level breaks to lower-level breaks.
- * Align the new line with beginning of the expression at the same level
- on the previous line.
- * If the above rules lead to confusing code, just indent 8 spaces.
-* One variable declaration per a line.
-
-## Naming
-
-* Packages should be all lowercase.
- * Java code should be in `org.apache.,metron`, except for compatibility classes
-* Classes should be in mixed case.
-* Variables should be in camel case.
-* Constants should be in upper case.
http://git-wip-us.apache.org/repos/asf/metron/blob/9b671005/site/develop/index.md
----------------------------------------------------------------------
diff --git a/site/develop/index.md b/site/develop/index.md
deleted file mode 100644
index 7db04fa..0000000
--- a/site/develop/index.md
+++ /dev/null
@@ -1,61 +0,0 @@
----
-layout: page
-title: Developing
----
-
-Information about the Metron project that is most important for
-developers working on the project. The project has created
-[bylaws](bylaws.html) for itself.
-
-## Project Members
-
-Name | Apache Id | Role
-:---------------------- | :------------- | :---
-Owen O'Malley | omalley | PMC
-Jim Baker | jimbaker | PMC
-Mark Bittmann | mbittmann | PMC
-Sheetal Dolas | sheetal_dolas | PMC
-P. Taylor Goetz | ptgoetz | PMC
-Brad Kolarov | billie | PMC
-Dave Hirko | dbhirko | PMC
-Larry McCay | lmccay | PMC
-Charles Porter | cporter | PMC
-James Sirota | jsirota | PMC
-Casey Stella | cestella | PMC
-Vinod Kumar Vavilapalli | vinodkv | PMC
-George Vetticaden | gvetticaden | PMC
-
-## Mailing Lists
-
-There are several development mailing lists for Metron
-
-* [dev@metron.apache.org](mailto:dev@metron.apache.org) - Development discussions
- with archive [here](https://mail-archives.apache.org/mod_mbox/metron-dev/)
-* [issues@metron.apache.org](mailto:issues@metron.apache.org) - Bug tracking
- with archive [here](https://mail-archives.apache.org/mod_mbox/metron-issues/)
-* [commits@metron.apache.org](mailto:commits@metron.apache.org) - Git tracking
- with archive [here](https://mail-archives.apache.org/mod_mbox/metron-commits/)
-
-You can subscribe to the lists by sending email to
-*list*-subscribe@metron.apache.org and unsubscribe by sending email to
-*list*-unsubscribe@metron.apache.org.
-
-## Source code
-
-Metron uses git for version control. Get the source code:
-
-`% git clone https://git-wip-us.apache.org/repos/asf/metron.git`
-
-The important branches are:
-
-* [master](https://github.com/apache/metron/tree/master) -
- The trunk for all developement
-* [asf-site](https://github.com/apache/metron/tree/asf-site) -
- The pages that are deployed to https://metron.apache.org/
-
-Please check our [coding guidelines](/develop/coding.html).
-
-## Reviews
-
-All code must be +1'ed by a committer other than the author prior to its
-commit.
[26/51] [abbrv] metron git commit: METRON-1771 Update REST endpoints
to support eventually consistent UI updates (merrimanr) closes
apache/metron#1190
Posted by ot...@apache.org.
METRON-1771 Update REST endpoints to support eventually consistent UI updates (merrimanr) closes apache/metron#1190
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/de533063
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/de533063
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/de533063
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: de533063c26c8a2462dbfff33228a949c4a96c97
Parents: 7e222fa
Author: merrimanr <me...@gmail.com>
Authored: Thu Oct 4 17:16:18 2018 -0500
Committer: rmerriman <me...@gmail.com>
Committed: Thu Oct 4 17:16:18 2018 -0500
----------------------------------------------------------------------
metron-interface/metron-rest/README.md | 16 ++
.../rest/controller/MetaAlertController.java | 18 +-
.../rest/controller/UpdateController.java | 31 ++-
.../metron/rest/service/MetaAlertService.java | 12 +-
.../metron/rest/service/UpdateService.java | 9 +-
.../rest/service/impl/MetaAlertServiceImpl.java | 18 +-
.../rest/service/impl/UpdateServiceImpl.java | 18 +-
.../MetaAlertControllerIntegrationTest.java | 152 +++++++-------
.../elasticsearch/dao/ElasticsearchDao.java | 32 +--
.../dao/ElasticsearchMetaAlertDao.java | 37 ++--
.../dao/ElasticsearchMetaAlertUpdateDao.java | 55 ++---
.../dao/ElasticsearchUpdateDao.java | 42 ++--
.../dao/ElasticsearchMetaAlertDaoTest.java | 27 ++-
.../dao/ElasticsearchUpdateDaoTest.java | 52 +++++
.../apache/metron/indexing/dao/HBaseDao.java | 31 +--
.../metron/indexing/dao/MultiIndexDao.java | 138 ++++++++-----
.../dao/metaalert/MetaAlertCreateResponse.java | 40 ----
.../dao/metaalert/MetaAlertUpdateDao.java | 20 +-
.../AbstractLuceneMetaAlertUpdateDao.java | 81 +++++++-
.../metron/indexing/dao/update/UpdateDao.java | 25 ++-
.../InMemoryMetaAlertRetrieveLatestDao.java | 49 +++++
.../metron/indexing/dao/HBaseDaoTest.java | 41 ++++
.../apache/metron/indexing/dao/InMemoryDao.java | 18 +-
.../indexing/dao/InMemoryMetaAlertDao.java | 199 +++++--------------
.../dao/InMemoryMetaAlertUpdateDao.java | 91 +++++++++
.../metron/indexing/dao/MultiIndexDaoTest.java | 96 +++++++++
.../metron/indexing/dao/UpdateDaoTest.java | 74 +++++++
.../indexing/dao/UpdateIntegrationTest.java | 138 +++++++------
.../dao/metaalert/MetaAlertIntegrationTest.java | 119 ++++++++---
.../AbstractLuceneMetaAlertUpdateDaoTest.java | 56 ++++--
.../integration/HBaseDaoIntegrationTest.java | 7 +-
.../org/apache/metron/solr/dao/SolrDao.java | 28 +--
.../metron/solr/dao/SolrMetaAlertDao.java | 49 +++--
.../metron/solr/dao/SolrMetaAlertUpdateDao.java | 38 ++--
.../apache/metron/solr/dao/SolrUpdateDao.java | 36 ++--
.../metron/solr/dao/SolrMetaAlertDaoTest.java | 21 +-
.../metron/solr/dao/SolrUpdateDaoTest.java | 19 +-
37 files changed, 1206 insertions(+), 727 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index 2c216d1..2ce9522 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -336,6 +336,8 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| [ `GET /api/v1/storm/supervisors`](#get-apiv1stormsupervisors)|
| [ `PATCH /api/v1/update/patch`](#patch-apiv1updatepatch)|
| [ `PUT /api/v1/update/replace`](#put-apiv1updatereplace)|
+| [ `POST /api/v1/update/add/comment`](#put-apiv1updateaddcomment)|
+| [ `POST /api/v1/update/remove/comment`](#put-apiv1updateremovecomment)|
| [ `GET /api/v1/user`](#get-apiv1user)|
### `POST /api/v1/alerts/ui/escalate`
@@ -963,6 +965,20 @@ Request and Response objects are JSON formatted. The JSON schemas are available
```
* Returns:
* 200 - Current user
+
+### `POST /api/v1/update/add/comment`
+ * Description: Add a comment to an alert
+ * Input:
+ * request - Comment add request
+ * Returns:
+ * 200 - Returns the complete alert document with comments added.
+
+### `POST /api/v1/update/remove/comment`
+ * Description: Remove a comment from an alert
+ * Input:
+ * request - Comment remove request
+ * Returns:
+ * 200 - Returns the complete alert document with comments removed.
### `GET /api/v1/user`
* Description: Retrieves the current user
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/MetaAlertController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/MetaAlertController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/MetaAlertController.java
index d42403a..69b1779 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/MetaAlertController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/MetaAlertController.java
@@ -24,8 +24,8 @@ import io.swagger.annotations.ApiResponse;
import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
import org.apache.metron.indexing.dao.metaalert.MetaAlertAddRemoveRequest;
import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
-import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.service.MetaAlertService;
import org.springframework.beans.factory.annotation.Autowired;
@@ -57,9 +57,9 @@ public class MetaAlertController {
@ApiOperation(value = "Creates a new meta alert from a list of existing alerts. "
+ "The meta alert status will initially be set to 'ACTIVE' and summary statistics "
+ "will be computed from the list of alerts. A list of groups included in the request are also added to the meta alert.")
- @ApiResponse(message = "The GUID of the new meta alert", code = 200)
+ @ApiResponse(message = "Returns the complete document of the created metaalert.", code = 200)
@RequestMapping(value = "/create", method = RequestMethod.POST)
- ResponseEntity<MetaAlertCreateResponse> create(
+ ResponseEntity<Document> create(
@ApiParam(name = "createRequest", value = "Meta alert create request which includes a list of alert "
+ "get requests and a list of custom groups used to annotate a meta alert", required = true)
@RequestBody final MetaAlertCreateRequest createRequest
@@ -68,9 +68,9 @@ public class MetaAlertController {
}
@ApiOperation(value = "Adds an alert to an existing meta alert. An alert will not be added if it is already contained in a meta alert.")
- @ApiResponse(message = "Returns 'true' if the alert was added and 'false' if the meta alert did not change.", code = 200)
+ @ApiResponse(message = "Returns the complete metaalert document with the alerts added.", code = 200)
@RequestMapping(value = "/add/alert", method = RequestMethod.POST)
- ResponseEntity<Boolean> addAlertsToMetaAlert(
+ ResponseEntity<Document> addAlertsToMetaAlert(
@ApiParam(name = "metaAlertAddRemoveRequest", value = "Meta alert add request which includes a meta alert GUID and list of alert get requests", required = true)
@RequestBody final MetaAlertAddRemoveRequest metaAlertAddRemoveRequest
) throws RestException {
@@ -78,9 +78,9 @@ public class MetaAlertController {
}
@ApiOperation(value = "Removes an alert from an existing meta alert. If the alert to be removed is not in a meta alert, 'false' will be returned.")
- @ApiResponse(message = "Returns 'true' if the alert was removed and 'false' if the meta alert did not change.", code = 200)
+ @ApiResponse(message = "Returns the complete metaalert document with the alerts removed.", code = 200)
@RequestMapping(value = "/remove/alert", method = RequestMethod.POST)
- ResponseEntity<Boolean> removeAlertsFromMetaAlert(
+ ResponseEntity<Document> removeAlertsFromMetaAlert(
@ApiParam(name = "metaAlertAddRemoveRequest", value = "Meta alert remove request which includes a meta alert GUID and list of alert get requests", required = true)
@RequestBody final MetaAlertAddRemoveRequest metaAlertAddRemoveRequest
) throws RestException {
@@ -88,9 +88,9 @@ public class MetaAlertController {
}
@ApiOperation(value = "Updates the status of a meta alert to either 'ACTIVE' or 'INACTIVE'.")
- @ApiResponse(message = "Returns 'true' if the status changed and 'false' if it did not.", code = 200)
+ @ApiResponse(message = "Returns the complete metaalert document with the updated status.", code = 200)
@RequestMapping(value = "/update/status/{guid}/{status}", method = RequestMethod.POST)
- ResponseEntity<Boolean> updateMetaAlertStatus(
+ ResponseEntity<Document> updateMetaAlertStatus(
final @ApiParam(name = "guid", value = "Meta alert GUID", required = true)
@PathVariable String guid,
final @ApiParam(name = "status", value = "Meta alert status with a value of either 'ACTIVE' or 'INACTIVE'", required = true)
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java
index 609442b..5550358 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java
@@ -21,6 +21,7 @@ import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import io.swagger.annotations.ApiResponse;
import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
+import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
@@ -42,52 +43,48 @@ public class UpdateController {
private UpdateService service;
@ApiOperation(value = "Update a document with a patch")
- @ApiResponse(message = "Nothing", code = 200)
+ @ApiResponse(message = "Returns the complete patched document.", code = 200)
@RequestMapping(value = "/patch", method = RequestMethod.PATCH)
- ResponseEntity<Void> patch(
+ ResponseEntity<Document> patch(
final @ApiParam(name = "request", value = "Patch request", required = true)
@RequestBody
PatchRequest request
) throws RestException {
try {
- service.patch(request);
+ return new ResponseEntity<>(service.patch(request), HttpStatus.OK);
} catch (OriginalNotFoundException e) {
return new ResponseEntity<>(HttpStatus.NOT_FOUND);
}
- return new ResponseEntity<>(HttpStatus.OK);
}
@ApiOperation(value = "Replace a document with a full replacement")
- @ApiResponse(message = "Nothing", code = 200)
+ @ApiResponse(message = "Returns the complete replaced document.", code = 200)
@RequestMapping(value = "/replace", method = RequestMethod.POST)
- ResponseEntity<Void> replace(
+ ResponseEntity<Document> replace(
final @ApiParam(name = "request", value = "Replacement request", required = true)
@RequestBody
ReplaceRequest request
) throws RestException {
- service.replace(request);
- return new ResponseEntity<>(HttpStatus.OK);
+ return new ResponseEntity<>(service.replace(request), HttpStatus.OK);
}
@ApiOperation(value = "Add a comment to an alert")
- @ApiResponse(message = "Nothing", code = 200)
+ @ApiResponse(message = "Returns the complete alert document with comments added.", code = 200)
@RequestMapping(value = "/add/comment", method = RequestMethod.POST)
- ResponseEntity<Void> addCommentToAlert(
+ ResponseEntity<Document> addCommentToAlert(
@RequestBody @ApiParam(name = "request", value = "Comment add request", required = true) final
CommentAddRemoveRequest request
) throws RestException {
- service.addComment(request);
- return new ResponseEntity<>(HttpStatus.OK);
+ return new ResponseEntity<>(service.addComment(request), HttpStatus.OK);
}
- @ApiOperation(value = "Remove a comment to an alert")
- @ApiResponse(message = "Nothing", code = 200)
+ @ApiOperation(value = "Remove a comment from an alert")
+ @ApiResponse(message = "Returns the complete alert document with comments removed.", code = 200)
@RequestMapping(value = "/remove/comment", method = RequestMethod.POST)
- ResponseEntity<Void> removeCommentFromAlert(
+ ResponseEntity<Document> removeCommentFromAlert(
@RequestBody @ApiParam(name = "request", value = "Comment remove request", required = true) final
CommentAddRemoveRequest request
) throws RestException {
- service.removeComment(request);
- return new ResponseEntity<>(HttpStatus.OK);
+ return new ResponseEntity<>(service.removeComment(request), HttpStatus.OK);
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/MetaAlertService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/MetaAlertService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/MetaAlertService.java
index e8abaf3..4ebebb6 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/MetaAlertService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/MetaAlertService.java
@@ -18,24 +18,22 @@
package org.apache.metron.rest.service;
-import java.io.IOException;
-import java.util.Collection;
import org.apache.metron.indexing.dao.metaalert.MetaAlertAddRemoveRequest;
import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
-import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.rest.RestException;
public interface MetaAlertService {
- MetaAlertCreateResponse create(MetaAlertCreateRequest createRequest) throws RestException;
+ Document create(MetaAlertCreateRequest createRequest) throws RestException;
SearchResponse getAllMetaAlertsForAlert(String guid) throws RestException;
- boolean addAlertsToMetaAlert(MetaAlertAddRemoveRequest metaAlertAddRemoveRequest) throws RestException;
+ Document addAlertsToMetaAlert(MetaAlertAddRemoveRequest metaAlertAddRemoveRequest) throws RestException;
- boolean removeAlertsFromMetaAlert(MetaAlertAddRemoveRequest metaAlertAddRemoveRequest) throws RestException;
+ Document removeAlertsFromMetaAlert(MetaAlertAddRemoveRequest metaAlertAddRemoveRequest) throws RestException;
- boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status) throws RestException;
+ Document updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status) throws RestException;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java
index bd59f39..19b3485 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java
@@ -18,6 +18,7 @@
package org.apache.metron.rest.service;
import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
+import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
@@ -25,8 +26,8 @@ import org.apache.metron.rest.RestException;
public interface UpdateService {
- void patch(PatchRequest request) throws RestException, OriginalNotFoundException;
- void replace(ReplaceRequest request) throws RestException;
- void addComment(CommentAddRemoveRequest request) throws RestException;
- void removeComment(CommentAddRemoveRequest request) throws RestException;
+ Document patch(PatchRequest request) throws RestException, OriginalNotFoundException;
+ Document replace(ReplaceRequest request) throws RestException;
+ Document addComment(CommentAddRemoveRequest request) throws RestException;
+ Document removeComment(CommentAddRemoveRequest request) throws RestException;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/MetaAlertServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/MetaAlertServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/MetaAlertServiceImpl.java
index 3f9b3e4..bd8419f 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/MetaAlertServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/MetaAlertServiceImpl.java
@@ -23,11 +23,11 @@ import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertAddRemoveRequest;
import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
-import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
import org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.service.MetaAlertService;
import org.springframework.beans.factory.annotation.Autowired;
@@ -47,7 +47,7 @@ public class MetaAlertServiceImpl implements MetaAlertService {
}
@Override
- public MetaAlertCreateResponse create(MetaAlertCreateRequest createRequest) throws RestException {
+ public Document create(MetaAlertCreateRequest createRequest) throws RestException {
try {
return dao.createMetaAlert(createRequest);
} catch (InvalidCreateException | IOException e) {
@@ -65,25 +65,25 @@ public class MetaAlertServiceImpl implements MetaAlertService {
}
@Override
- public boolean addAlertsToMetaAlert(MetaAlertAddRemoveRequest metaAlertAddRemoveRequest) throws RestException {
+ public Document addAlertsToMetaAlert(MetaAlertAddRemoveRequest metaAlertAddRemoveRequest) throws RestException {
try {
return dao.addAlertsToMetaAlert(metaAlertAddRemoveRequest.getMetaAlertGuid(), metaAlertAddRemoveRequest.getAlerts());
- } catch (IOException ioe) {
- throw new RestException(ioe.getMessage(), ioe);
+ } catch (IOException | IllegalStateException e) {
+ throw new RestException(e.getMessage(), e);
}
}
@Override
- public boolean removeAlertsFromMetaAlert(MetaAlertAddRemoveRequest metaAlertAddRemoveRequest) throws RestException {
+ public Document removeAlertsFromMetaAlert(MetaAlertAddRemoveRequest metaAlertAddRemoveRequest) throws RestException {
try {
return dao.removeAlertsFromMetaAlert(metaAlertAddRemoveRequest.getMetaAlertGuid(), metaAlertAddRemoveRequest.getAlerts());
- } catch (IOException ioe) {
- throw new RestException(ioe.getMessage(), ioe);
+ } catch (IOException | IllegalStateException e) {
+ throw new RestException(e.getMessage(), e);
}
}
@Override
- public boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status)
+ public Document updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status)
throws RestException {
try {
return dao.updateMetaAlertStatus(metaAlertGuid, status);
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java
index 49490fd..63dd4c0 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java
@@ -19,6 +19,7 @@ package org.apache.metron.rest.service.impl;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
+import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
@@ -43,37 +44,36 @@ public class UpdateServiceImpl implements UpdateService {
@Override
- public void patch(PatchRequest request) throws RestException, OriginalNotFoundException {
+ public Document patch(PatchRequest request) throws RestException, OriginalNotFoundException {
try {
- dao.patch(dao, request, Optional.of(System.currentTimeMillis()));
+ return dao.patch(dao, request, Optional.of(System.currentTimeMillis()));
} catch (Exception e) {
-
throw new RestException(e.getMessage(), e);
}
}
@Override
- public void replace(ReplaceRequest request) throws RestException {
+ public Document replace(ReplaceRequest request) throws RestException {
try {
- dao.replace(request, Optional.of(System.currentTimeMillis()));
+ return dao.replace(request, Optional.of(System.currentTimeMillis()));
} catch (Exception e) {
throw new RestException(e.getMessage(), e);
}
}
@Override
- public void addComment(CommentAddRemoveRequest request) throws RestException {
+ public Document addComment(CommentAddRemoveRequest request) throws RestException {
try {
- dao.addCommentToAlert(request);
+ return dao.addCommentToAlert(request);
} catch (Exception e) {
throw new RestException(e.getMessage(), e);
}
}
@Override
- public void removeComment(CommentAddRemoveRequest request) throws RestException {
+ public Document removeComment(CommentAddRemoveRequest request) throws RestException {
try {
- dao.removeCommentFromAlert(request);
+ return dao.removeCommentFromAlert(request);
} catch (Exception e) {
throw new RestException(e.getMessage(), e);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
index b216990..8e3abe5 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
@@ -19,6 +19,10 @@
package org.apache.metron.rest.controller;
import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
@@ -29,15 +33,13 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
import com.google.common.collect.ImmutableMap;
import java.util.ArrayList;
-import java.util.Arrays;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.curator.framework.CuratorFramework;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.InMemoryMetaAlertDao;
import org.apache.metron.indexing.dao.SearchIntegrationTest;
import org.apache.metron.indexing.dao.metaalert.MetaAlertAddRemoveRequest;
-import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
-import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.rest.service.MetaAlertService;
import org.junit.After;
@@ -96,8 +98,8 @@ public class MetaAlertControllerIntegrationTest extends DaoControllerTest {
/**
* [
- *{"guid":"meta_1","metron_alert":[{"guid":"bro_1"}],"average":"5.0","min":"5.0","median":"5.0","max":"5.0","count":"1.0","sum":"5.0"},
- *{"guid":"meta_2","metron_alert":[{"guid":"bro_1"},{"guid":"bro_2"},{"guid":"snort_1"}],"average":"5.0","min":"0.0","median":"5.0","max":"10.0","count":"3.0","sum":"15.0"}
+ *{"guid":"meta_1","metron_alert":[{"guid":"bro_1", "source.type":"bro"}],"average":"5.0","min":"5.0","median":"5.0","max":"5.0","count":"1.0","sum":"5.0", "status":"active"},
+ *{"guid":"meta_2","metron_alert":[{"guid":"bro_1", "source.type":"bro"},{"guid":"bro_2", "source.type":"bro"},{"guid":"snort_1", "source.type":"snort"}],"average":"5.0","min":"0.0","median":"5.0","max":"10.0","count":"3.0","sum":"15.0"}
* ]
*/
@Multiline
@@ -163,58 +165,32 @@ public class MetaAlertControllerIntegrationTest extends DaoControllerTest {
.andExpect(jsonPath("$.results[0].source.count").value(3.0))
.andExpect(jsonPath("$.results[1].source.guid").value("meta_1"))
.andExpect(jsonPath("$.results[1].source.count").value(1.0));
+ }
- result = this.mockMvc.perform(
- post(metaalertUrl + "/create")
- .with(httpBasic(user, password)).with(csrf())
- .contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))
- .content(create));
- result.andExpect(status().isOk());
-
- // Test that we can find the newly created meta alert by the sub alerts
- guid = "bro_1";
- result = this.mockMvc.perform(
- post(metaalertUrl + "/searchByAlert")
- .with(httpBasic(user, password)).with(csrf())
- .contentType(MediaType.parseMediaType("text/plain;charset=UTF-8"))
- .content(guid));
- result.andExpect(status().isOk())
- .andExpect(
- content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$.total").value(3))
- .andExpect(jsonPath("$.results[0].source.guid").value("meta_3"))
- .andExpect(jsonPath("$.results[0].source.count").value(2.0))
- .andExpect(jsonPath("$.results[1].source.guid").value("meta_2"))
- .andExpect(jsonPath("$.results[1].source.count").value(3.0))
- .andExpect(jsonPath("$.results[2].source.guid").value("meta_1"))
- .andExpect(jsonPath("$.results[2].source.count").value(1.0));
-
- guid = "snort_2";
- result = this.mockMvc.perform(
- post(metaalertUrl + "/searchByAlert")
- .with(httpBasic(user, password)).with(csrf())
- .contentType(MediaType.parseMediaType("text/plain;charset=UTF-8"))
- .content(guid));
+ @Test
+ public void shouldCreateMetaAlert() throws Exception {
+ ResultActions result = this.mockMvc.perform(
+ post(metaalertUrl + "/create")
+ .with(httpBasic(user, password)).with(csrf())
+ .contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))
+ .content(create));
result.andExpect(status().isOk())
- .andExpect(
- content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$.total").value(1))
- .andExpect(jsonPath("$.results[0].source.guid").value("meta_3"))
- .andExpect(jsonPath("$.results[0].source.count").value(2.0));
+ .andExpect(jsonPath("$.guid", notNullValue()))
+ .andExpect(jsonPath("$.timestamp", greaterThan(0L)))
+ .andExpect(jsonPath("$.sensorType").value(MetaAlertConstants.METAALERT_TYPE))
+ .andExpect(jsonPath("$.document.timestamp", greaterThan(0L)))
+ .andExpect(jsonPath("$.document['source.type']").value(MetaAlertConstants.METAALERT_TYPE))
+ .andExpect(jsonPath("$.document.status").value("active"))
+ .andExpect(jsonPath("$.document.groups[0]").value("group_one"))
+ .andExpect(jsonPath("$.document.groups[1]").value("group_two"))
+ .andExpect(jsonPath("$.document.metron_alert[0].guid").value("bro_1"))
+ .andExpect(jsonPath("$.document.metron_alert[1].guid").value("snort_2"));
}
@Test
public void shouldAddRemoveAlerts() throws Exception {
- MetaAlertCreateRequest metaAlertCreateRequest = new MetaAlertCreateRequest();
- metaAlertCreateRequest.setGroups(Arrays.asList("group_one", "group_two"));
- metaAlertCreateRequest.setAlerts(new ArrayList<GetRequest>() {{
- add(new GetRequest("bro_1", "bro", "bro_index_2017.01.01.01"));
- add(new GetRequest("snort_2", "snort", "snort_index_2017.01.01.01"));
- }});
- MetaAlertCreateResponse metaAlertCreateResponse = metaAlertService.create(metaAlertCreateRequest);
-
MetaAlertAddRemoveRequest addRequest = new MetaAlertAddRemoveRequest();
- addRequest.setMetaAlertGuid(metaAlertCreateResponse.getGuid());
+ addRequest.setMetaAlertGuid("meta_1");
addRequest.setAlerts(new ArrayList<GetRequest>() {{
add(new GetRequest("bro_2", "bro", "bro_index_2017.01.01.01"));
add(new GetRequest("bro_3", "bro", "bro_index_2017.01.01.01"));
@@ -225,10 +201,17 @@ public class MetaAlertControllerIntegrationTest extends DaoControllerTest {
.with(httpBasic(user, password)).with(csrf())
.contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))
.content(JSONUtils.INSTANCE.toJSON(addRequest, false)));
- result.andExpect(status().isOk()).andExpect(content().string("true"));
+ result.andExpect(status().isOk())
+ .andExpect(jsonPath("$.guid").value("meta_1"))
+ .andExpect(jsonPath("$.sensorType").value(MetaAlertConstants.METAALERT_TYPE))
+ .andExpect(jsonPath("$.document.metron_alert[0].guid").value("bro_1"))
+ .andExpect(jsonPath("$.document.metron_alert[1].guid").value("bro_2"))
+ .andExpect(jsonPath("$.document.metron_alert[2].metaalerts").value("meta_1"))
+ .andExpect(jsonPath("$.document.metron_alert[2].guid").value("bro_3"))
+ .andExpect(jsonPath("$.document.metron_alert[2].metaalerts").value("meta_1"));
MetaAlertAddRemoveRequest addDuplicateRequest = new MetaAlertAddRemoveRequest();
- addDuplicateRequest.setMetaAlertGuid(metaAlertCreateResponse.getGuid());
+ addDuplicateRequest.setMetaAlertGuid("meta_1");
addDuplicateRequest.setAlerts(new ArrayList<GetRequest>() {{
add(new GetRequest("bro_1", "bro"));
}});
@@ -238,10 +221,17 @@ public class MetaAlertControllerIntegrationTest extends DaoControllerTest {
.with(httpBasic(user, password)).with(csrf())
.contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))
.content(JSONUtils.INSTANCE.toJSON(addDuplicateRequest, false)));
- result.andExpect(status().isOk()).andExpect(content().string("false"));
+ result.andExpect(status().isOk())
+ .andExpect(jsonPath("$.guid").value("meta_1"))
+ .andExpect(jsonPath("$.sensorType").value(MetaAlertConstants.METAALERT_TYPE))
+ .andExpect(jsonPath("$.document.metron_alert[0].guid").value("bro_1"))
+ .andExpect(jsonPath("$.document.metron_alert[1].guid").value("bro_2"))
+ .andExpect(jsonPath("$.document.metron_alert[2].metaalerts").value("meta_1"))
+ .andExpect(jsonPath("$.document.metron_alert[2].guid").value("bro_3"))
+ .andExpect(jsonPath("$.document.metron_alert[2].metaalerts").value("meta_1"));
MetaAlertAddRemoveRequest removeRequest = new MetaAlertAddRemoveRequest();
- removeRequest.setMetaAlertGuid(metaAlertCreateResponse.getGuid());
+ removeRequest.setMetaAlertGuid("meta_1");
removeRequest.setAlerts(new ArrayList<GetRequest>() {{
add(new GetRequest("bro_2", "bro"));
add(new GetRequest("bro_3", "bro"));
@@ -252,12 +242,16 @@ public class MetaAlertControllerIntegrationTest extends DaoControllerTest {
.with(httpBasic(user, password)).with(csrf())
.contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))
.content(JSONUtils.INSTANCE.toJSON(removeRequest, false)));
- result.andExpect(status().isOk()).andExpect(content().string("true"));
+ result.andExpect(status().isOk())
+ .andExpect(jsonPath("$.guid").value("meta_1"))
+ .andExpect(jsonPath("$.sensorType").value(MetaAlertConstants.METAALERT_TYPE))
+ .andExpect(jsonPath("$.document.metron_alert.*", hasSize(equalTo(1))))
+ .andExpect(jsonPath("$.document.metron_alert[0].guid").value("bro_1"));
MetaAlertAddRemoveRequest removeMissingRequest = new MetaAlertAddRemoveRequest();
- addRequest.setMetaAlertGuid(metaAlertCreateResponse.getGuid());
+ removeMissingRequest.setMetaAlertGuid("meta_1");
removeMissingRequest.setAlerts(new ArrayList<GetRequest>() {{
- add(new GetRequest("bro_1", "bro"));
+ add(new GetRequest("bro_2", "bro"));
}});
result = this.mockMvc.perform(
@@ -265,31 +259,47 @@ public class MetaAlertControllerIntegrationTest extends DaoControllerTest {
.with(httpBasic(user, password)).with(csrf())
.contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))
.content(JSONUtils.INSTANCE.toJSON(removeMissingRequest, false)));
- result.andExpect(status().isOk()).andExpect(content().string("false"));
+ result.andExpect(status().isOk())
+ .andExpect(jsonPath("$.guid").value("meta_1"))
+ .andExpect(jsonPath("$.sensorType").value(MetaAlertConstants.METAALERT_TYPE))
+ .andExpect(jsonPath("$.document.metron_alert.*", hasSize(equalTo(1))))
+ .andExpect(jsonPath("$.document.metron_alert[0].guid").value("bro_1"));
+
+ MetaAlertAddRemoveRequest emptyMetaAlertRequest = new MetaAlertAddRemoveRequest();
+ emptyMetaAlertRequest.setMetaAlertGuid("meta_1");
+ emptyMetaAlertRequest.setAlerts(new ArrayList<GetRequest>() {{
+ add(new GetRequest("bro_1", "bro"));
+ }});
+
+ result = this.mockMvc.perform(
+ post(metaalertUrl + "/remove/alert")
+ .with(httpBasic(user, password)).with(csrf())
+ .contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))
+ .content(JSONUtils.INSTANCE.toJSON(emptyMetaAlertRequest, false)));
+ result.andExpect(status().isInternalServerError())
+ .andExpect(jsonPath("$.message").value("Removing these alerts will result in an empty meta alert. Empty meta alerts are not allowed."))
+ .andExpect(jsonPath("$.fullMessage").value("IllegalStateException: Removing these alerts will result in an empty meta alert. Empty meta alerts are not allowed."));
}
@Test
public void shouldUpdateStatus() throws Exception {
- MetaAlertCreateRequest metaAlertCreateRequest = new MetaAlertCreateRequest();
- metaAlertCreateRequest.setGroups(Arrays.asList("group_one", "group_two"));
- metaAlertCreateRequest.setAlerts(new ArrayList<GetRequest>() {{
- add(new GetRequest("bro_1", "bro", "bro_index_2017.01.01.01"));
- add(new GetRequest("snort_2", "snort", "snort_index_2017.01.01.01"));
- }});
-
- MetaAlertCreateResponse metaAlertCreateResponse = metaAlertService.create(metaAlertCreateRequest);
-
ResultActions result = this.mockMvc.perform(
- post(metaalertUrl + "/update/status/" + metaAlertCreateResponse.getGuid() + "/inactive")
+ post(metaalertUrl + "/update/status/meta_2/inactive")
.with(httpBasic(user, password)).with(csrf())
.contentType(MediaType.parseMediaType("application/json;charset=UTF-8")));
- result.andExpect(status().isOk()).andExpect(content().string("true"));
+ result.andExpect(status().isOk())
+ .andExpect(jsonPath("$.guid").value("meta_2"))
+ .andExpect(jsonPath("$.sensorType").value(MetaAlertConstants.METAALERT_TYPE))
+ .andExpect(jsonPath("$.document.status").value("inactive"));
result = this.mockMvc.perform(
- post(metaalertUrl + "/update/status/" + metaAlertCreateResponse.getGuid() + "/inactive")
+ post(metaalertUrl + "/update/status/meta_2/active")
.with(httpBasic(user, password)).with(csrf())
.contentType(MediaType.parseMediaType("application/json;charset=UTF-8")));
- result.andExpect(status().isOk()).andExpect(content().string("false"));
+ result.andExpect(status().isOk())
+ .andExpect(jsonPath("$.guid").value("meta_2"))
+ .andExpect(jsonPath("$.sensorType").value(MetaAlertConstants.METAALERT_TYPE))
+ .andExpect(jsonPath("$.document.status").value("active"));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index 59f25f0..9f6e1a1 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -138,34 +138,34 @@ public class ElasticsearchDao implements IndexDao {
}
@Override
- public void update(Document update, Optional<String> index) throws IOException {
- updateDao.update(update, index);
+ public Document update(Document update, Optional<String> index) throws IOException {
+ return updateDao.update(update, index);
}
@Override
- public void batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
- updateDao.batchUpdate(updates);
+ public Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
+ return updateDao.batchUpdate(updates);
}
@Override
- public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request, Optional<Long> timestamp)
+ public Document patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request, Optional<Long> timestamp)
throws OriginalNotFoundException, IOException {
- updateDao.patch(retrieveLatestDao, request, timestamp);
+ return updateDao.patch(retrieveLatestDao, request, timestamp);
}
@Override
- public void replace(ReplaceRequest request, Optional<Long> timestamp) throws IOException {
- updateDao.replace(request, timestamp);
+ public Document replace(ReplaceRequest request, Optional<Long> timestamp) throws IOException {
+ return updateDao.replace(request, timestamp);
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
- updateDao.addCommentToAlert(request);
+ public Document addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ return updateDao.addCommentToAlert(request);
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
- updateDao.removeCommentFromAlert(request);
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ return updateDao.removeCommentFromAlert(request);
}
@Override
@@ -179,13 +179,13 @@ public class ElasticsearchDao implements IndexDao {
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
- this.updateDao.addCommentToAlert(request, latest);
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ return this.updateDao.addCommentToAlert(request, latest);
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
- this.updateDao.removeCommentFromAlert(request, latest);
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ return this.updateDao.removeCommentFromAlert(request, latest);
}
protected Optional<String> getIndexName(String guid, String sensorType) {
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index 55123a5..fc0b20c 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -26,7 +26,6 @@ import org.apache.metron.indexing.dao.RetrieveLatestDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
-import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
import org.apache.metron.indexing.dao.metaalert.MetaAlertDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
import org.apache.metron.indexing.dao.search.FieldType;
@@ -182,25 +181,25 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
}
@Override
- public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request)
+ public Document createMetaAlert(MetaAlertCreateRequest request)
throws InvalidCreateException, IOException {
return metaAlertUpdateDao.createMetaAlert(request);
}
@Override
- public boolean addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
+ public Document addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
throws IOException {
return metaAlertUpdateDao.addAlertsToMetaAlert(metaAlertGuid, alertRequests);
}
@Override
- public boolean removeAlertsFromMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
+ public Document removeAlertsFromMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
throws IOException {
return metaAlertUpdateDao.removeAlertsFromMetaAlert(metaAlertGuid, alertRequests);
}
@Override
- public boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status)
+ public Document updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status)
throws IOException {
return metaAlertUpdateDao.updateMetaAlertStatus(metaAlertGuid, status);
}
@@ -216,40 +215,40 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
}
@Override
- public void update(Document update, Optional<String> index) throws IOException {
- metaAlertUpdateDao.update(update, index);
+ public Document update(Document update, Optional<String> index) throws IOException {
+ return metaAlertUpdateDao.update(update, index);
}
@Override
- public void batchUpdate(Map<Document, Optional<String>> updates) {
- metaAlertUpdateDao.batchUpdate(updates);
+ public Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) {
+ return metaAlertUpdateDao.batchUpdate(updates);
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
- indexDao.addCommentToAlert(request);
+ public Document addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ return indexDao.addCommentToAlert(request);
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
- indexDao.removeCommentFromAlert(request);
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ return indexDao.removeCommentFromAlert(request);
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
- indexDao.addCommentToAlert(request, latest);
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ return indexDao.addCommentToAlert(request, latest);
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
- indexDao.removeCommentFromAlert(request, latest);
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ return indexDao.removeCommentFromAlert(request, latest);
}
@Override
- public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
+ public Document patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
Optional<Long> timestamp)
throws OriginalNotFoundException, IOException {
- metaAlertUpdateDao.patch(retrieveLatestDao, request, timestamp);
+ return metaAlertUpdateDao.patch(retrieveLatestDao, request, timestamp);
}
public void setPageSize(int pageSize) {
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
index bb79b7a..3b67891 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
@@ -30,7 +30,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.lucene.search.join.ScoreMode;
import org.apache.metron.common.Constants;
@@ -38,8 +37,6 @@ import org.apache.metron.elasticsearch.utils.ElasticsearchUtils;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
-import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
-import org.apache.metron.indexing.dao.metaalert.MetaAlertDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertRetrieveLatestDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
import org.apache.metron.indexing.dao.metaalert.MetaScores;
@@ -82,7 +79,7 @@ public class ElasticsearchMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpda
@Override
@SuppressWarnings("unchecked")
- public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request)
+ public Document createMetaAlert(MetaAlertCreateRequest request)
throws InvalidCreateException, IOException {
List<GetRequest> alertRequests = request.getAlerts();
if (request.getAlerts().isEmpty()) {
@@ -134,39 +131,14 @@ public class ElasticsearchMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpda
// Kick off any updates.
update(updates);
- MetaAlertCreateResponse createResponse = new MetaAlertCreateResponse();
- createResponse.setCreated(true);
- createResponse.setGuid(metaAlert.getGuid());
- return createResponse;
+ return metaAlert;
} catch (IOException ioe) {
throw new InvalidCreateException("Unable to create meta alert", ioe);
}
}
- /**
- * Adds alerts to a metaalert, based on a list of GetRequests provided for retrieval.
- * @param metaAlertGuid The GUID of the metaalert to be given new children.
- * @param alertRequests GetRequests for the appropriate alerts to add.
- * @return True if metaalert is modified, false otherwise.
- */
- public boolean addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
- throws IOException {
-
- Document metaAlert = retrieveLatestDao
- .getLatest(metaAlertGuid, MetaAlertConstants.METAALERT_TYPE);
- if (MetaAlertStatus.ACTIVE.getStatusString()
- .equals(metaAlert.getDocument().get(MetaAlertConstants.STATUS_FIELD))) {
- Iterable<Document> alerts = retrieveLatestDao.getAllLatest(alertRequests);
- Map<Document, Optional<String>> updates = buildAddAlertToMetaAlertUpdates(metaAlert, alerts);
- update(updates);
- return updates.size() != 0;
- } else {
- throw new IllegalStateException("Adding alerts to an INACTIVE meta alert is not allowed");
- }
- }
-
@Override
- public void update(Document update, Optional<String> index) throws IOException {
+ public Document update(Document update, Optional<String> index) throws IOException {
if (MetaAlertConstants.METAALERT_TYPE.equals(update.getSensorType())) {
// We've been passed an update to the meta alert.
throw new UnsupportedOperationException("Meta alerts cannot be directly updated");
@@ -195,29 +167,31 @@ public class ElasticsearchMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpda
// Run the alert's update
elasticsearchDao.batchUpdate(updates);
+
+ return update;
}
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
- getUpdateDao().addCommentToAlert(request);
+ public Document addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ return getUpdateDao().addCommentToAlert(request);
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
- getUpdateDao().removeCommentFromAlert(request);
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ return getUpdateDao().removeCommentFromAlert(request);
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest)
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest)
throws IOException {
- getUpdateDao().addCommentToAlert(request, latest);
+ return getUpdateDao().addCommentToAlert(request, latest);
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
throws IOException {
- getUpdateDao().removeCommentFromAlert(request, latest);
+ return getUpdateDao().removeCommentFromAlert(request, latest);
}
/**
@@ -243,12 +217,11 @@ public class ElasticsearchMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpda
}
- protected boolean replaceAlertInMetaAlert(Document metaAlert, Document alert) {
+ protected void replaceAlertInMetaAlert(Document metaAlert, Document alert) {
boolean metaAlertUpdated = removeAlertsFromMetaAlert(metaAlert,
Collections.singleton(alert.getGuid()));
if (metaAlertUpdated) {
addAlertsToMetaAlert(metaAlert, Collections.singleton(alert));
}
- return metaAlertUpdated;
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
index f2b08d2..6843ac7 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
@@ -60,7 +60,7 @@ public class ElasticsearchUpdateDao implements UpdateDao {
}
@Override
- public void update(Document update, Optional<String> index) throws IOException {
+ public Document update(Document update, Optional<String> index) throws IOException {
String indexPostfix = ElasticsearchUtils
.getIndexFormat(accessConfig.getGlobalConfigSupplier().get()).format(new Date());
String sensorType = update.getSensorType();
@@ -79,10 +79,11 @@ public class ElasticsearchUpdateDao implements UpdateDao {
} catch (Exception e) {
throw new IOException(e.getMessage(), e);
}
+ return update;
}
@Override
- public void batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
+ public Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
String indexPostfix = ElasticsearchUtils
.getIndexFormat(accessConfig.getGlobalConfigSupplier().get()).format(new Date());
@@ -108,20 +109,22 @@ public class ElasticsearchUpdateDao implements UpdateDao {
throw new IOException(
"ElasticsearchDao upsert failed: " + bulkResponse.buildFailureMessage());
}
+ return updates;
}
@Override
@SuppressWarnings("unchecked")
- public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ public Document addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
- addCommentToAlert(request, latest);
+ return addCommentToAlert(request, latest);
}
@Override
@SuppressWarnings("unchecked")
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
- if (latest == null) {
- return;
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ if (latest == null || latest.getDocument() == null) {
+ throw new IOException(String.format("Unable to add comment. Document with guid %s cannot be found.",
+ request.getGuid()));
}
List<Map<String, Object>> commentsField = (List<Map<String, Object>>) latest.getDocument()
.getOrDefault(COMMENTS_FIELD, new ArrayList<>());
@@ -133,25 +136,30 @@ public class ElasticsearchUpdateDao implements UpdateDao {
Document newVersion = new Document(latest);
newVersion.getDocument().put(COMMENTS_FIELD, originalComments);
- update(newVersion, Optional.empty());
+ return update(newVersion, Optional.empty());
}
@Override
@SuppressWarnings("unchecked")
- public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
- removeCommentFromAlert(request, latest);
+ return removeCommentFromAlert(request, latest);
}
@Override
@SuppressWarnings("unchecked")
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
- if (latest == null) {
- return;
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ if (latest == null || latest.getDocument() == null) {
+ throw new IOException(String.format("Unable to remove comment. Document with guid %s cannot be found.",
+ request.getGuid()));
}
- List<Map<String, Object>> commentsField = (List<Map<String, Object>>) latest.getDocument()
- .getOrDefault(COMMENTS_FIELD, new ArrayList<>());
- List<Map<String, Object>> originalComments = new ArrayList<>(commentsField);
+ List<Map<String, Object>> commentMap = (List<Map<String, Object>>) latest.getDocument().get(COMMENTS_FIELD);
+ // Can't remove anything if there's nothing there
+ if (commentMap == null) {
+ throw new IOException(String.format("Unable to remove comment. Document with guid %s has no comments.",
+ request.getGuid()));
+ }
+ List<Map<String, Object>> originalComments = new ArrayList<>(commentMap);
List<AlertComment> alertComments = new ArrayList<>();
for (Map<String, Object> commentRaw : originalComments) {
@@ -170,7 +178,7 @@ public class ElasticsearchUpdateDao implements UpdateDao {
newVersion.getDocument().remove(COMMENTS_FIELD);
}
- update(newVersion, Optional.empty());
+ return update(newVersion, Optional.empty());
}
protected String getIndexName(Document update, Optional<String> index, String indexPostFix) {
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
index b1da2a4..cabb992 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
@@ -19,8 +19,6 @@
package org.apache.metron.elasticsearch.dao;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -44,13 +42,6 @@ import org.apache.metron.indexing.dao.update.Document;
import org.elasticsearch.index.IndexNotFoundException;
import org.junit.Test;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
@@ -88,11 +79,13 @@ public class ElasticsearchMetaAlertDaoTest {
}
@Override
- public void update(Document update, Optional<String> index) {
+ public Document update(Document update, Optional<String> index) {
+ return update;
}
@Override
- public void batchUpdate(Map<Document, Optional<String>> updates) {
+ public Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) {
+ return updates;
}
@Override
@@ -101,19 +94,23 @@ public class ElasticsearchMetaAlertDaoTest {
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) {
+ public Document addCommentToAlert(CommentAddRemoveRequest request) {
+ return null;
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request) {
+ return null;
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ return null;
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ return null;
}
};
ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao();
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDaoTest.java
new file mode 100644
index 0000000..3b48a60
--- /dev/null
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDaoTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.elasticsearch.dao;
+
+import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.UpdateDaoTest;
+import org.apache.metron.indexing.dao.update.UpdateDao;
+import org.elasticsearch.client.transport.TransportClient;
+import org.junit.Before;
+
+import static org.mockito.Mockito.mock;
+
+/**
+ * This class returns the ElasticsearchUpdateDao implementation to be used in UpdateDaoTest. UpdateDaoTest contains a
+ * common set of tests that all Dao implementations must pass.
+ */
+public class ElasticsearchUpdateDaoTest extends UpdateDaoTest {
+
+ private TransportClient client;
+ private AccessConfig accessConfig;
+ private ElasticsearchRetrieveLatestDao retrieveLatestDao;
+ private ElasticsearchUpdateDao updateDao;
+
+ @Before
+ public void setup() {
+ client = mock(TransportClient.class);
+ accessConfig = new AccessConfig();
+ retrieveLatestDao = mock(ElasticsearchRetrieveLatestDao.class);
+ updateDao = new ElasticsearchUpdateDao(client, accessConfig, retrieveLatestDao);
+ }
+
+ @Override
+ public UpdateDao getUpdateDao() {
+ return updateDao;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
index 6c646de..71d0544 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
@@ -238,13 +238,14 @@ public class HBaseDao implements IndexDao {
}
@Override
- public synchronized void update(Document update, Optional<String> index) throws IOException {
+ public synchronized Document update(Document update, Optional<String> index) throws IOException {
Put put = buildPut(update);
getTableInterface().put(put);
+ return update;
}
@Override
- public void batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
+ public Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
List<Put> puts = new ArrayList<>();
for (Map.Entry<Document, Optional<String>> updateEntry : updates.entrySet()) {
Document update = updateEntry.getKey();
@@ -253,6 +254,7 @@ public class HBaseDao implements IndexDao {
puts.add(put);
}
getTableInterface().put(puts);
+ return updates;
}
protected Get buildGet(GetRequest getRequest) throws IOException {
@@ -280,16 +282,17 @@ public class HBaseDao implements IndexDao {
@Override
@SuppressWarnings("unchecked")
- public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ public Document addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
Document latest = getLatest(request.getGuid(), request.getSensorType());
- addCommentToAlert(request, latest);
+ return addCommentToAlert(request, latest);
}
@Override
@SuppressWarnings("unchecked")
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
if (latest == null || latest.getDocument() == null) {
- throw new IOException("Unable to add comment to document that doesn't exist");
+ throw new IOException(String.format("Unable to add comment. Document with guid %s cannot be found.",
+ request.getGuid()));
}
List<Map<String, Object>> comments = (List<Map<String, Object>>) latest.getDocument()
@@ -309,28 +312,30 @@ public class HBaseDao implements IndexDao {
Document newVersion = new Document(latest);
newVersion.getDocument().put(COMMENTS_FIELD, commentsMap);
- update(newVersion, Optional.empty());
+ return update(newVersion, Optional.empty());
}
@Override
@SuppressWarnings("unchecked")
- public void removeCommentFromAlert(CommentAddRemoveRequest request)
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request)
throws IOException {
Document latest = getLatest(request.getGuid(), request.getSensorType());
- removeCommentFromAlert(request, latest);
+ return removeCommentFromAlert(request, latest);
}
@Override
@SuppressWarnings("unchecked")
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
throws IOException {
if (latest == null || latest.getDocument() == null) {
- throw new IOException("Unable to remove comment document that doesn't exist");
+ throw new IOException(String.format("Unable to remove comment. Document with guid %s cannot be found.",
+ request.getGuid()));
}
List<Map<String, Object>> commentMap = (List<Map<String, Object>>) latest.getDocument().get(COMMENTS_FIELD);
// Can't remove anything if there's nothing there
if (commentMap == null) {
- return;
+ throw new IOException(String.format("Unable to remove comment. Document with guid %s has no comments.",
+ request.getGuid()));
}
List<Map<String, Object>> originalComments = new ArrayList<>(commentMap);
List<AlertComment> comments = new ArrayList<>();
@@ -349,6 +354,6 @@ public class HBaseDao implements IndexDao {
newVersion.getDocument().remove(COMMENTS_FIELD);
}
- update(newVersion, Optional.empty());
+ return update(newVersion, Optional.empty());
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
index 420c775..c3e2108 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -58,7 +57,7 @@ public class MultiIndexDao implements IndexDao {
}
@Override
- public void update(final Document update, Optional<String> index) throws IOException {
+ public Document update(final Document update, Optional<String> index) throws IOException {
List<String> exceptions =
indices.parallelStream().map(dao -> {
try {
@@ -71,10 +70,11 @@ public class MultiIndexDao implements IndexDao {
if(exceptions.size() > 0) {
throw new IOException(Joiner.on("\n").join(exceptions));
}
+ return update;
}
@Override
- public void batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
+ public Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
List<String> exceptions =
indices.parallelStream().map(dao -> {
try {
@@ -87,6 +87,7 @@ public class MultiIndexDao implements IndexDao {
if (exceptions.size() > 0) {
throw new IOException(Joiner.on("\n").join(exceptions));
}
+ return updates;
}
@Override
@@ -101,51 +102,62 @@ public class MultiIndexDao implements IndexDao {
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ public Document addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
Document latest = getLatest(request.getGuid(), request.getSensorType());
- addCommentToAlert(request, latest);
+ return addCommentToAlert(request, latest);
}
-
+ /**
+ * Adds comments to an alert. Updates are written to each Dao in parallel with the assumption that all updates
+ * are identical. The first update to be applied is returned as the current version of the alert with comments added.
+ * @param request Request to add comments
+ * @param latest The latest version of the alert the comments will be added to.
+ * @return The complete alert document with comments added.
+ * @throws IOException
+ */
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
- List<String> exceptions =
- indices.parallelStream().map(dao -> {
- try {
- dao.addCommentToAlert(request, latest);
- return null;
- } catch (Throwable e) {
- return dao.getClass() + ": " + e.getMessage() + "\n" + ExceptionUtils.getStackTrace(e);
- }
- }).filter(Objects::nonNull).collect(Collectors.toList());
- if (exceptions.size() > 0) {
- throw new IOException(Joiner.on("\n").join(exceptions));
- }
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ List<DocumentContainer> output =
+ indices.parallelStream().map(dao -> {
+ try {
+ return new DocumentContainer(dao.addCommentToAlert(request, latest));
+ } catch (Throwable e) {
+ return new DocumentContainer(e);
+ }
+ }).collect(Collectors.toList());
+
+ return getLatestDocument(output);
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
Document latest = getLatest(request.getGuid(), request.getSensorType());
- removeCommentFromAlert(request, latest);
+ return removeCommentFromAlert(request, latest);
}
+ /**
+ * Removes comments from an alert. Updates are written to each Dao in parallel with the assumption that all updates
+ * are identical. The first update to be applied is returned as the current version of the alert with comments removed.
+ * @param request Request to remove comments
+ * @param latest The latest version of the alert the comments will be removed from.
+ * @return The complete alert document with comments removed.
+ * @throws IOException
+ */
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
- List<String> exceptions =
- indices.parallelStream().map(dao -> {
- try {
- dao.removeCommentFromAlert(request, latest);
- return null;
- } catch (Throwable e) {
- return dao.getClass() + ": " + e.getMessage() + "\n" + ExceptionUtils.getStackTrace(e);
- }
- }).filter(Objects::nonNull).collect(Collectors.toList());
- if (exceptions.size() > 0) {
- throw new IOException(Joiner.on("\n").join(exceptions));
- }
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ List<DocumentContainer> output =
+ indices.parallelStream().map(dao -> {
+ try {
+ return new DocumentContainer(dao.removeCommentFromAlert(request, latest));
+ } catch (Throwable e) {
+ return new DocumentContainer(e);
+ }
+ }).collect(Collectors.toList());
+
+ return getLatestDocument(output);
}
- private static class DocumentContainer {
+ protected static class DocumentContainer {
private Optional<Document> d = Optional.empty();
private Optional<Throwable> t = Optional.empty();
public DocumentContainer(Document d) {
@@ -214,7 +226,6 @@ public class MultiIndexDao implements IndexDao {
@Override
public Document getLatest(final String guid, String sensorType) throws IOException {
- Document ret = null;
List<DocumentContainer> output =
indices.parallelStream().map(dao -> {
try {
@@ -224,25 +235,7 @@ public class MultiIndexDao implements IndexDao {
}
}).collect(Collectors.toList());
- List<String> error = new ArrayList<>();
- for(DocumentContainer dc : output) {
- if(dc.getException().isPresent()) {
- Throwable e = dc.getException().get();
- error.add(e.getMessage() + "\n" + ExceptionUtils.getStackTrace(e));
- }
- else {
- if(dc.getDocument().isPresent()) {
- Document d = dc.getDocument().get();
- if(ret == null || ret.getTimestamp() < d.getTimestamp()) {
- ret = d;
- }
- }
- }
- }
- if(error.size() > 0) {
- throw new IOException(Joiner.on("\n").join(error));
- }
- return ret;
+ return getLatestDocument(output);
}
@Override
@@ -282,4 +275,39 @@ public class MultiIndexDao implements IndexDao {
public List<IndexDao> getIndices() {
return indices;
}
+
+ /**
+ * Returns the most recent {@link Document} from a list of {@link DocumentContainer}s.
+ *
+ * @param documentContainers A list of containers; each retrieved from a separate index.
+ * @return The latest {@link Document} found.
+ * @throws IOException If any of the {@link DocumentContainer}s contain an exception.
+ */
+ private Document getLatestDocument(List<DocumentContainer> documentContainers) throws IOException {
+ Document latestDocument = null;
+ List<String> error = new ArrayList<>();
+
+ for(DocumentContainer dc : documentContainers) {
+ if(dc.getException().isPresent()) {
+ // collect each exception; multiple can occur, one in each index
+ Throwable e = dc.getException().get();
+ error.add(e.getMessage() + "\n" + ExceptionUtils.getStackTrace(e));
+
+ } else if(dc.getDocument().isPresent()) {
+ Document d = dc.getDocument().get();
+ // is this the latest document so far?
+ if(latestDocument == null || latestDocument.getTimestamp() < d.getTimestamp()) {
+ latestDocument = d;
+ }
+
+ } else {
+ // no document was found in the index
+ }
+ }
+ if(error.size() > 0) {
+ // report all of the errors encountered
+ throw new IOException(Joiner.on("\n").join(error));
+ }
+ return latestDocument;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java
deleted file mode 100644
index 0bdf332..0000000
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.indexing.dao.metaalert;
-
-public class MetaAlertCreateResponse {
- private boolean created;
- private String guid;
-
- public boolean isCreated() {
- return created;
- }
-
- public void setCreated(boolean created) {
- this.created = created;
- }
-
- public String getGuid() {
- return guid;
- }
-
- public void setGuid(String guid) {
- this.guid = guid;
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertUpdateDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertUpdateDao.java
index f4374b4..4b22656 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertUpdateDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertUpdateDao.java
@@ -59,31 +59,31 @@ public interface MetaAlertUpdateDao extends UpdateDao {
* retrieved using the DAO abstractions.
*
* @param request A request object containing get requests for alerts to be added and a list of groups
- * @return A response indicating success or failure along with the GUID of the new meta alert
+ * @return The complete document of the created metaalert.
* @throws InvalidCreateException If a malformed create request is provided
* @throws IOException If a problem occurs during communication
*/
- MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request)
+ Document createMetaAlert(MetaAlertCreateRequest request)
throws InvalidCreateException, IOException;
/**
* Adds alerts to a metaalert, based on a list of GetRequests provided for retrieval.
* @param metaAlertGuid The GUID of the metaalert to be given new children.
* @param alertRequests GetRequests for the appropriate alerts to add.
- * @return True if metaalert is modified, false otherwise.
+ * @return The complete metaalert document with the alerts added.
*/
- boolean addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
- throws IOException;
+ Document addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
+ throws IOException, IllegalStateException;
/**
* Removes alerts from a metaalert
* @param metaAlertGuid The metaalert guid to be affected.
* @param alertRequests A list of GetReqests that will provide the alerts to remove
- * @return True if there are updates, false otherwise
+ * @return The complete metaalert document with the alerts removed.
* @throws IOException If an error is thrown during retrieal.
*/
- boolean removeAlertsFromMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
- throws IOException;
+ Document removeAlertsFromMetaAlert(String metaAlertGuid, List<GetRequest> alertRequests)
+ throws IOException, IllegalStateException;
/**
* Removes a metaalert link from a given alert. An nonexistent link performs no change.
@@ -115,10 +115,10 @@ public interface MetaAlertUpdateDao extends UpdateDao {
*
* @param metaAlertGuid The GUID of the meta alert
* @param status A status value of 'active' or 'inactive'
- * @return True or false depending on if the status was changed
+ * @return The complete metaalert document with the updated status.
* @throws IOException if an error occurs during the update.
*/
- boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status)
+ Document updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status)
throws IOException;
/**
[28/51] [abbrv] metron git commit: METRON-1808 Add Ansible created
pyc to gitignore (justinleet) closes apache/metron#1228
Posted by ot...@apache.org.
METRON-1808 Add Ansible created pyc to gitignore (justinleet) closes apache/metron#1228
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/891bf495
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/891bf495
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/891bf495
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 891bf4952ab8eceabcb44659e7498d62005b6aa2
Parents: feb9153
Author: justinleet <ju...@gmail.com>
Authored: Fri Oct 5 16:14:20 2018 -0400
Committer: leet <le...@apache.org>
Committed: Fri Oct 5 16:14:20 2018 -0400
----------------------------------------------------------------------
metron-deployment/.gitignore | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/891bf495/metron-deployment/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/.gitignore b/metron-deployment/.gitignore
index 6038b6b..f3d4a2a 100644
--- a/metron-deployment/.gitignore
+++ b/metron-deployment/.gitignore
@@ -1,3 +1,4 @@
keys/
.vagrant
*.retry
+*.pyc
[03/51] [abbrv] metron git commit: Merge remote-tracking branch
'apache/master' into feature/METRON-1699-create-batch-profiler
Posted by ot...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java
----------------------------------------------------------------------
diff --cc metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java
index fc94afa,0000000..3f16edd
mode 100644,000000..100644
--- a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java
+++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileBuilderBoltTest.java
@@@ -1,356 -1,0 +1,378 @@@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.storm;
+
+import org.apache.metron.common.configuration.profiler.ProfileConfig;
+import org.apache.metron.common.configuration.profiler.ProfilerConfigurations;
+import org.apache.metron.profiler.MessageDistributor;
+import org.apache.metron.profiler.MessageRoute;
+import org.apache.metron.profiler.ProfileMeasurement;
+import org.apache.metron.profiler.storm.integration.MessageBuilder;
+import org.apache.metron.test.bolt.BaseBoltTest;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.topology.base.BaseWindowedBolt;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
+import org.apache.storm.windowing.TupleWindow;
+import org.json.simple.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
++import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests the ProfileBuilderBolt.
+ */
+public class ProfileBuilderBoltTest extends BaseBoltTest {
+
+ private JSONObject message1;
+ private JSONObject message2;
+ private ProfileConfig profile1;
+ private ProfileConfig profile2;
+ private ProfileMeasurementEmitter emitter;
+ private ManualFlushSignal flushSignal;
+ private ProfileMeasurement measurement;
+
+ @Before
+ public void setup() throws Exception {
+
+ message1 = new MessageBuilder()
+ .withField("ip_src_addr", "10.0.0.1")
+ .withField("value", "22")
+ .build();
+
+ message2 = new MessageBuilder()
+ .withField("ip_src_addr", "10.0.0.2")
+ .withField("value", "22")
+ .build();
+
+ profile1 = new ProfileConfig()
+ .withProfile("profile1")
+ .withForeach("ip_src_addr")
+ .withInit("x", "0")
+ .withUpdate("x", "x + 1")
+ .withResult("x");
+
+ profile2 = new ProfileConfig()
+ .withProfile("profile2")
+ .withForeach("ip_src_addr")
+ .withInit(Collections.singletonMap("x", "0"))
+ .withUpdate(Collections.singletonMap("x", "x + 1"))
+ .withResult("x");
+
+ measurement = new ProfileMeasurement()
+ .withEntity("entity1")
+ .withProfileName("profile1")
+ .withPeriod(1000, 500, TimeUnit.MILLISECONDS)
+ .withProfileValue(22);
+
+ flushSignal = new ManualFlushSignal();
+ flushSignal.setFlushNow(false);
+ }
+
+ /**
+ * The bolt should extract a message and timestamp from a tuple and
+ * pass that to a {@code MessageDistributor}.
+ */
+ @Test
+ public void testExtractMessage() throws Exception {
+
+ ProfileBuilderBolt bolt = createBolt();
+
+ // create a mock
+ MessageDistributor distributor = mock(MessageDistributor.class);
+ bolt.withMessageDistributor(distributor);
+
+ // create a tuple
+ final long timestamp1 = 100000000L;
+ Tuple tuple1 = createTuple("entity1", message1, profile1, timestamp1);
+
+ // execute the bolt
+ TupleWindow tupleWindow = createWindow(tuple1);
+ bolt.execute(tupleWindow);
+
+ // the message should have been extracted from the tuple and passed to the MessageDistributor
+ verify(distributor).distribute(any(MessageRoute.class), any());
+ }
+
+
+ /**
+ * If the {@code FlushSignal} tells the bolt to flush, it should flush the {@code MessageDistributor}
+ * and emit the {@code ProfileMeasurement} values from all active profiles.
+ */
+ @Test
+ public void testFlushActiveProfiles() throws Exception {
+
+ ProfileBuilderBolt bolt = createBolt();
+
+ // create a mock that returns the profile measurement above
+ MessageDistributor distributor = mock(MessageDistributor.class);
+ when(distributor.flush()).thenReturn(Collections.singletonList(measurement));
+ bolt.withMessageDistributor(distributor);
+
+ // signal the bolt to flush
+ flushSignal.setFlushNow(true);
+
+ // execute the bolt
+ Tuple tuple1 = createTuple("entity1", message1, profile1, 1000L);
+ TupleWindow tupleWindow = createWindow(tuple1);
+ bolt.execute(tupleWindow);
+
+ // a profile measurement should be emitted by the bolt
+ List<ProfileMeasurement> measurements = getProfileMeasurements(outputCollector, 1);
+ assertEquals(1, measurements.size());
+ assertEquals(measurement, measurements.get(0));
+ }
+
+ /**
+ * If the {@code FlushSignal} tells the bolt NOT to flush, nothing should be emitted.
+ */
+ @Test
+ public void testDoNotFlushActiveProfiles() throws Exception {
+
+ ProfileBuilderBolt bolt = createBolt();
+
+ // create a mock where flush() returns the profile measurement above
+ MessageDistributor distributor = mock(MessageDistributor.class);
+ when(distributor.flush()).thenReturn(Collections.singletonList(measurement));
+ bolt.withMessageDistributor(distributor);
+
+ // there is no flush signal
+ flushSignal.setFlushNow(false);
+
+ // execute the bolt
+ Tuple tuple1 = createTuple("entity1", message1, profile1, 1000L);
+ TupleWindow tupleWindow = createWindow(tuple1);
+ bolt.execute(tupleWindow);
+
+ // nothing should have been emitted
+ getProfileMeasurements(outputCollector, 0);
+ }
+
+ /**
+ * Expired profiles should be flushed regularly, even if no input telemetry
+ * has been received.
+ */
+ @Test
+ public void testFlushExpiredProfiles() throws Exception {
+
+ ProfileBuilderBolt bolt = createBolt();
+
+ // create a mock where flushExpired() returns the profile measurement above
+ MessageDistributor distributor = mock(MessageDistributor.class);
+ when(distributor.flushExpired()).thenReturn(Collections.singletonList(measurement));
+ bolt.withMessageDistributor(distributor);
+
+ // execute test by flushing expired profiles. this is normally triggered by a timer task.
+ bolt.flushExpired();
+
+ // a profile measurement should be emitted by the bolt
+ List<ProfileMeasurement> measurements = getProfileMeasurements(outputCollector, 1);
+ assertEquals(1, measurements.size());
+ assertEquals(measurement, measurements.get(0));
+ }
+
+ /**
+ * A {@link ProfileMeasurement} is built for each profile/entity pair. The measurement should be emitted to each
+ * destination defined by the profile. By default, a profile uses both Kafka and HBase as destinations.
+ */
+ @Test
+ public void testEmitters() throws Exception {
+
+ // defines the zk configurations accessible from the bolt
+ ProfilerConfigurations configurations = new ProfilerConfigurations();
+ configurations.updateGlobalConfig(Collections.emptyMap());
+
+ // create the bolt with 3 destinations
+ ProfileBuilderBolt bolt = (ProfileBuilderBolt) new ProfileBuilderBolt()
+ .withProfileTimeToLive(30, TimeUnit.MINUTES)
+ .withPeriodDuration(10, TimeUnit.MINUTES)
+ .withMaxNumberOfRoutes(Long.MAX_VALUE)
+ .withZookeeperClient(client)
+ .withZookeeperCache(cache)
+ .withEmitter(new TestEmitter("destination1"))
+ .withEmitter(new TestEmitter("destination2"))
+ .withEmitter(new TestEmitter("destination3"))
+ .withProfilerConfigurations(configurations)
+ .withTumblingWindow(new BaseWindowedBolt.Duration(10, TimeUnit.MINUTES));
+ bolt.prepare(new HashMap<>(), topologyContext, outputCollector);
+
+ // signal the bolt to flush
+ bolt.withFlushSignal(flushSignal);
+ flushSignal.setFlushNow(true);
+
+ // execute the bolt
+ Tuple tuple1 = createTuple("entity", message1, profile1, System.currentTimeMillis());
+ TupleWindow window = createWindow(tuple1);
+ bolt.execute(window);
+
+ // validate measurements emitted to each
+ verify(outputCollector, times(1)).emit(eq("destination1"), any());
+ verify(outputCollector, times(1)).emit(eq("destination2"), any());
+ verify(outputCollector, times(1)).emit(eq("destination3"), any());
+ }
+
++ @Test
++ public void testExceptionWhenFlushingExpiredProfiles() throws Exception {
++ // create an emitter that will throw an exception when emit() called
++ ProfileMeasurementEmitter badEmitter = mock(ProfileMeasurementEmitter.class);
++ doThrow(new RuntimeException("flushExpired() should catch this exception"))
++ .when(badEmitter)
++ .emit(any(), any());
++
++ // create a distributor that will return a profile measurement
++ MessageDistributor distributor = mock(MessageDistributor.class);
++ when(distributor.flushExpired()).thenReturn(Collections.singletonList(measurement));
++
++ // the bolt will use the bad emitter when flushExpired() is called
++ ProfileBuilderBolt bolt = (ProfileBuilderBolt) new ProfileBuilderBolt()
++ .withEmitter(badEmitter)
++ .withMessageDistributor(distributor);
++
++ // the exception thrown by the emitter should not bubble up
++ bolt.flushExpired();
++ }
++
+ /**
+ * Retrieves the ProfileMeasurement(s) (if any) that have been emitted.
+ *
+ * @param collector The Storm output collector.
+ * @param expected The number of measurements expected.
+ * @return A list of ProfileMeasurement(s).
+ */
+ private List<ProfileMeasurement> getProfileMeasurements(OutputCollector collector, int expected) {
+
+ // the 'streamId' is defined by the DestinationHandler being used by the bolt
+ final String streamId = emitter.getStreamId();
+
+ // capture the emitted tuple(s)
+ ArgumentCaptor<Values> argCaptor = ArgumentCaptor.forClass(Values.class);
+ verify(collector, times(expected))
+ .emit(eq(streamId), argCaptor.capture());
+
+ // return the profile measurements that were emitted
+ return argCaptor.getAllValues()
+ .stream()
+ .map(val -> (ProfileMeasurement) val.get(0))
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Create a tuple that will contain the message, the entity name, and profile definition.
+ * @param entity The entity name
+ * @param message The telemetry message.
+ * @param profile The profile definition.
+ */
+ private Tuple createTuple(String entity, JSONObject message, ProfileConfig profile, long timestamp) {
+
+ Tuple tuple = mock(Tuple.class);
+ when(tuple.getValueByField(eq(ProfileSplitterBolt.MESSAGE_TUPLE_FIELD))).thenReturn(message);
+ when(tuple.getValueByField(eq(ProfileSplitterBolt.TIMESTAMP_TUPLE_FIELD))).thenReturn(timestamp);
+ when(tuple.getValueByField(eq(ProfileSplitterBolt.ENTITY_TUPLE_FIELD))).thenReturn(entity);
+ when(tuple.getValueByField(eq(ProfileSplitterBolt.PROFILE_TUPLE_FIELD))).thenReturn(profile);
+
+ return tuple;
+ }
+
+ /**
+ * Create a ProfileBuilderBolt to test.
+ * @return A {@link ProfileBuilderBolt} to test.
+ */
+ private ProfileBuilderBolt createBolt() throws IOException {
+
+ // defines the zk configurations accessible from the bolt
+ ProfilerConfigurations configurations = new ProfilerConfigurations();
+ configurations.updateGlobalConfig(Collections.emptyMap());
+
+ emitter = new HBaseEmitter();
+ ProfileBuilderBolt bolt = (ProfileBuilderBolt) new ProfileBuilderBolt()
+ .withProfileTimeToLive(30, TimeUnit.MINUTES)
+ .withMaxNumberOfRoutes(Long.MAX_VALUE)
+ .withZookeeperClient(client)
+ .withZookeeperCache(cache)
+ .withEmitter(emitter)
+ .withProfilerConfigurations(configurations)
+ .withPeriodDuration(1, TimeUnit.MINUTES)
+ .withTumblingWindow(new BaseWindowedBolt.Duration(30, TimeUnit.SECONDS));
+ bolt.prepare(new HashMap<>(), topologyContext, outputCollector);
+
+ // set the flush signal AFTER calling 'prepare'
+ bolt.withFlushSignal(flushSignal);
+
+ return bolt;
+ }
+
+ /**
+ * Creates a mock TupleWindow containing multiple tuples.
+ * @param tuples The tuples to add to the window.
+ */
+ private TupleWindow createWindow(Tuple... tuples) {
+
+ TupleWindow window = mock(TupleWindow.class);
+ when(window.get()).thenReturn(Arrays.asList(tuples));
+ return window;
+ }
+
+ /**
+ * An implementation for testing purposes only.
+ */
+ private class TestEmitter implements ProfileMeasurementEmitter {
+
+ private String streamId;
+
+ public TestEmitter(String streamId) {
+ this.streamId = streamId;
+ }
+
+ @Override
+ public String getStreamId() {
+ return streamId;
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declareStream(getStreamId(), new Fields("measurement"));
+ }
+
+ @Override
+ public void emit(ProfileMeasurement measurement, OutputCollector collector) {
+ collector.emit(getStreamId(), new Values(measurement));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
----------------------------------------------------------------------
diff --cc metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
index 182600a,0000000..4389d42
mode 100644,000000..100644
--- a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
+++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
@@@ -1,421 -1,0 +1,478 @@@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.storm.integration;
+
+import org.adrianwalker.multilinestring.Multiline;
- import org.apache.hadoop.hbase.Cell;
- import org.apache.hadoop.hbase.client.Put;
- import org.apache.hadoop.hbase.util.Bytes;
++import org.apache.commons.io.FileUtils;
+import org.apache.metron.common.Constants;
- import org.apache.metron.common.utils.SerDeUtils;
+import org.apache.metron.hbase.mock.MockHBaseTableProvider;
+import org.apache.metron.hbase.mock.MockHTable;
+import org.apache.metron.integration.BaseIntegrationTest;
+import org.apache.metron.integration.ComponentRunner;
+import org.apache.metron.integration.UnableToStartException;
+import org.apache.metron.integration.components.FluxTopologyComponent;
+import org.apache.metron.integration.components.KafkaComponent;
+import org.apache.metron.integration.components.ZKServerComponent;
- import org.apache.metron.profiler.ProfileMeasurement;
- import org.apache.metron.profiler.hbase.ColumnBuilder;
- import org.apache.metron.profiler.hbase.RowKeyBuilder;
- import org.apache.metron.profiler.hbase.SaltyRowKeyBuilder;
- import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder;
++import org.apache.metron.profiler.client.stellar.FixedLookback;
++import org.apache.metron.profiler.client.stellar.GetProfile;
++import org.apache.metron.profiler.client.stellar.WindowLookback;
++import org.apache.metron.statistics.OnlineStatisticsProvider;
++import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor;
++import org.apache.metron.stellar.common.StellarStatefulExecutor;
++import org.apache.metron.stellar.dsl.Context;
++import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver;
+import org.apache.storm.Config;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
++import org.slf4j.Logger;
++import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.UnsupportedEncodingException;
- import java.util.ArrayList;
++import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
++import java.util.Collections;
++import java.util.HashMap;
+import java.util.List;
++import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import static com.google.code.tempusfugit.temporal.Duration.seconds;
+import static com.google.code.tempusfugit.temporal.Timeout.timeout;
+import static com.google.code.tempusfugit.temporal.WaitFor.waitOrTimeout;
- import static org.junit.Assert.assertArrayEquals;
++import static org.apache.metron.profiler.client.stellar.ProfilerClientConfig.PROFILER_COLUMN_FAMILY;
++import static org.apache.metron.profiler.client.stellar.ProfilerClientConfig.PROFILER_HBASE_TABLE;
++import static org.apache.metron.profiler.client.stellar.ProfilerClientConfig.PROFILER_HBASE_TABLE_PROVIDER;
++import static org.apache.metron.profiler.client.stellar.ProfilerClientConfig.PROFILER_PERIOD;
++import static org.apache.metron.profiler.client.stellar.ProfilerClientConfig.PROFILER_PERIOD_UNITS;
++import static org.apache.metron.profiler.client.stellar.ProfilerClientConfig.PROFILER_SALT_DIVISOR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * An integration test of the Profiler topology.
+ */
+public class ProfilerIntegrationTest extends BaseIntegrationTest {
+
++ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final String TEST_RESOURCES = "../../metron-analytics/metron-profiler-storm/src/test";
+ private static final String FLUX_PATH = "src/main/flux/profiler/remote.yaml";
+
+ public static final long startAt = 10;
+ public static final String entity = "10.0.0.1";
+
+ private static final String tableName = "profiler";
+ private static final String columnFamily = "P";
+ private static final String inputTopic = Constants.INDEXING_TOPIC;
+ private static final String outputTopic = "profiles";
+ private static final int saltDivisor = 10;
+
- private static final long windowLagMillis = TimeUnit.SECONDS.toMillis(1);
- private static final long windowDurationMillis = TimeUnit.SECONDS.toMillis(5);
- private static final long periodDurationMillis = TimeUnit.SECONDS.toMillis(10);
- private static final long profileTimeToLiveMillis = TimeUnit.SECONDS.toMillis(15);
++ private static final long periodDurationMillis = TimeUnit.SECONDS.toMillis(20);
++ private static final long windowLagMillis = TimeUnit.SECONDS.toMillis(10);
++ private static final long windowDurationMillis = TimeUnit.SECONDS.toMillis(10);
++ private static final long profileTimeToLiveMillis = TimeUnit.SECONDS.toMillis(20);
++
+ private static final long maxRoutesPerBolt = 100000;
+
- private static ColumnBuilder columnBuilder;
+ private static ZKServerComponent zkComponent;
+ private static FluxTopologyComponent fluxComponent;
+ private static KafkaComponent kafkaComponent;
+ private static ConfigUploadComponent configUploadComponent;
+ private static ComponentRunner runner;
+ private static MockHTable profilerTable;
+
+ private static String message1;
+ private static String message2;
+ private static String message3;
+
++ private StellarStatefulExecutor executor;
++
+ /**
+ * [
+ * org.apache.metron.profiler.ProfileMeasurement,
+ * org.apache.metron.profiler.ProfilePeriod,
+ * org.apache.metron.common.configuration.profiler.ProfileResult,
+ * org.apache.metron.common.configuration.profiler.ProfileResultExpressions,
+ * org.apache.metron.common.configuration.profiler.ProfileTriageExpressions,
+ * org.apache.metron.common.configuration.profiler.ProfilerConfig,
+ * org.apache.metron.common.configuration.profiler.ProfileConfig,
+ * org.json.simple.JSONObject,
++ * org.json.simple.JSONArray,
+ * java.util.LinkedHashMap,
+ * org.apache.metron.statistics.OnlineStatisticsProvider
+ * ]
+ */
+ @Multiline
+ private static String kryoSerializers;
+
- /**
- * The Profiler can generate profiles based on processing time. With processing time,
- * the Profiler builds profiles based on when the telemetry was processed.
- *
- * <p>Not defining a 'timestampField' within the Profiler configuration tells the Profiler
- * to use processing time.
- */
+ @Test
+ public void testProcessingTime() throws Exception {
++ uploadConfigToZookeeper(TEST_RESOURCES + "/config/zookeeper/processing-time-test");
+
- // upload the config to zookeeper
- uploadConfig(TEST_RESOURCES + "/config/zookeeper/processing-time-test");
-
- // start the topology and write test messages to kafka
++ // start the topology and write 3 test messages to kafka
+ fluxComponent.submitTopology();
-
- // the messages that will be applied to the profile
+ kafkaComponent.writeMessages(inputTopic, message1);
+ kafkaComponent.writeMessages(inputTopic, message2);
+ kafkaComponent.writeMessages(inputTopic, message3);
+
- // storm needs at least one message to close its event window
++ // retrieve the profile measurement using PROFILE_GET
++ String profileGetExpression = "PROFILE_GET('processing-time-test', '10.0.0.1', PROFILE_FIXED('5', 'MINUTES'))";
++ List<Integer> measurements = execute(profileGetExpression, List.class);
++
++ // need to keep checking for measurements until the profiler has flushed one out
+ int attempt = 0;
- while(profilerTable.getPutLog().size() == 0 && attempt++ < 10) {
++ while(measurements.size() == 0 && attempt++ < 10) {
+
- // sleep, at least beyond the current window
- Thread.sleep(windowDurationMillis + windowLagMillis);
++ // wait for the profiler to flush
++ long sleep = windowDurationMillis;
++ LOG.debug("Waiting {} millis for profiler to flush", sleep);
++ Thread.sleep(sleep);
+
- // send another message to help close the current event window
++ // write another message to advance time. this ensures we are testing the 'normal' flush mechanism.
++ // if we do not send additional messages to advance time, then it is the profile TTL mechanism which
++ // will ultimately flush the profile
+ kafkaComponent.writeMessages(inputTopic, message2);
++
++ // try again to retrieve the profile measurement using PROFILE_GET
++ measurements = execute(profileGetExpression, List.class);
+ }
+
- // validate what was flushed
- List<Integer> actuals = read(
- profilerTable.getPutLog(),
- columnFamily,
- columnBuilder.getColumnQualifier("value"),
- Integer.class);
- assertEquals(1, actuals.size());
- assertTrue(actuals.get(0) >= 3);
++ // expect to see only 1 measurement, but could be more (one for each period) depending on
++ // how long we waited for the flush to occur
++ assertTrue(measurements.size() > 0);
++
++ // the profile should have counted at least 3 messages; the 3 test messages that were sent.
++ // the count could be higher due to the test messages we sent to advance time.
++ assertTrue(measurements.get(0) >= 3);
+ }
+
- /**
- * The Profiler can generate profiles using event time. With event time processing,
- * the Profiler uses timestamps contained in the source telemetry.
- *
- * <p>Defining a 'timestampField' within the Profiler configuration tells the Profiler
- * from which field the timestamp should be extracted.
- */
+ @Test
- public void testEventTime() throws Exception {
-
- // upload the profiler config to zookeeper
- uploadConfig(TEST_RESOURCES + "/config/zookeeper/event-time-test");
++ public void testProcessingTimeWithTimeToLiveFlush() throws Exception {
++ uploadConfigToZookeeper(TEST_RESOURCES + "/config/zookeeper/processing-time-test");
+
- // start the topology and write test messages to kafka
++ // start the topology and write 3 test messages to kafka
+ fluxComponent.submitTopology();
+ kafkaComponent.writeMessages(inputTopic, message1);
+ kafkaComponent.writeMessages(inputTopic, message2);
+ kafkaComponent.writeMessages(inputTopic, message3);
+
- // wait until the profile is flushed
- waitOrTimeout(() -> profilerTable.getPutLog().size() > 0, timeout(seconds(90)));
++ // wait a bit beyond the window lag before writing another message. this allows storm's window manager to close
++ // the event window, which then lets the profiler processes the previous messages.
++ long sleep = windowLagMillis + periodDurationMillis;
++ LOG.debug("Waiting {} millis before sending message to close window", sleep);
++ Thread.sleep(sleep);
++ kafkaComponent.writeMessages(inputTopic, message3);
++
++ // retrieve the profile measurement using PROFILE_GET
++ String profileGetExpression = "PROFILE_GET('processing-time-test', '10.0.0.1', PROFILE_FIXED('5', 'MINUTES'))";
++ List<Integer> measurements = execute(profileGetExpression, List.class);
+
- List<Put> puts = profilerTable.getPutLog();
- assertEquals(1, puts.size());
++ // need to keep checking for measurements until the profiler has flushed one out
++ int attempt = 0;
++ while(measurements.size() == 0 && attempt++ < 10) {
++
++ // wait for the profiler to flush
++ sleep = windowDurationMillis;
++ LOG.debug("Waiting {} millis for profiler to flush", sleep);
++ Thread.sleep(sleep);
++
++ // do not write additional messages to advance time. this ensures that we are testing the "time to live"
++ // flush mechanism. the TTL setting defines when the profile will be flushed
++
++ // try again to retrieve the profile measurement
++ measurements = execute(profileGetExpression, List.class);
++ }
++
++ // expect to see only 1 measurement, but could be more (one for each period) depending on
++ // how long we waited for the flush to occur
++ assertTrue(measurements.size() > 0);
+
- // inspect the row key to ensure the profiler used event time correctly. the timestamp
- // embedded in the row key should match those in the source telemetry
- byte[] expectedRowKey = generateExpectedRowKey("event-time-test", entity, startAt);
- byte[] actualRowKey = puts.get(0).getRow();
- assertArrayEquals(failMessage(expectedRowKey, actualRowKey), expectedRowKey, actualRowKey);
++ // the profile should have counted 3 messages; the 3 test messages that were sent
++ assertEquals(3, measurements.get(0).intValue());
++ }
++
++ @Test
++ public void testEventTime() throws Exception {
++ uploadConfigToZookeeper(TEST_RESOURCES + "/config/zookeeper/event-time-test");
++
++ // start the topology and write test messages to kafka
++ fluxComponent.submitTopology();
++ List<String> messages = FileUtils.readLines(new File("src/test/resources/telemetry.json"));
++ kafkaComponent.writeMessages(inputTopic, messages);
++
++ long timestamp = System.currentTimeMillis();
++ LOG.debug("Attempting to close window period by sending message with timestamp = {}", timestamp);
++ kafkaComponent.writeMessages(inputTopic, getMessage("192.168.66.1", timestamp));
++ kafkaComponent.writeMessages(inputTopic, getMessage("192.168.138.158", timestamp));
++
++ // create the 'window' that looks up to 5 hours before the max timestamp contained in the test data
++ assign("maxTimestamp", "1530978728982L");
++ assign("window", "PROFILE_WINDOW('from 5 hours ago', maxTimestamp)");
++
++ // wait until the profile flushes both periods. the first period will flush immediately as subsequent messages
++ // advance time. the next period contains all of the remaining messages, so there are no other messages to
++ // advance time. because of this the next period only flushes after the time-to-live expires
++ waitOrTimeout(() -> profilerTable.getPutLog().size() >= 6, timeout(seconds(90)));
++ {
++ // there are 14 messages in the first period and 12 in the next where ip_src_addr = 192.168.66.1
++ List results = execute("PROFILE_GET('count-by-ip', '192.168.66.1', window)", List.class);
++ assertEquals(14, results.get(0));
++ assertEquals(12, results.get(1));
++ }
++ {
++ // there are 36 messages in the first period and 38 in the next where ip_src_addr = 192.168.138.158
++ List results = execute("PROFILE_GET('count-by-ip', '192.168.138.158', window)", List.class);
++ assertEquals(36, results.get(0));
++ assertEquals(38, results.get(1));
++ }
++ {
++ // in all there are 50 messages in the first period and 50 messages in the next
++ List results = execute("PROFILE_GET('total-count', 'total', window)", List.class);
++ assertEquals(50, results.get(0));
++ assertEquals(50, results.get(1));
++ }
+ }
+
+ /**
+ * The result produced by a Profile has to be serializable within Storm. If the result is not
+ * serializable the topology will crash and burn.
+ *
+ * This test ensures that if a profile returns a STATS object created using the STATS_INIT and
+ * STATS_ADD functions, that it can be correctly serialized and persisted.
+ */
+ @Test
+ public void testProfileWithStatsObject() throws Exception {
-
- // upload the profiler config to zookeeper
- uploadConfig(TEST_RESOURCES + "/config/zookeeper/profile-with-stats");
++ uploadConfigToZookeeper(TEST_RESOURCES + "/config/zookeeper/profile-with-stats");
+
+ // start the topology and write test messages to kafka
+ fluxComponent.submitTopology();
- kafkaComponent.writeMessages(inputTopic, message1);
- kafkaComponent.writeMessages(inputTopic, message2);
- kafkaComponent.writeMessages(inputTopic, message3);
++ List<String> messages = FileUtils.readLines(new File("src/test/resources/telemetry.json"));
++ kafkaComponent.writeMessages(inputTopic, messages);
+
+ // wait until the profile is flushed
+ waitOrTimeout(() -> profilerTable.getPutLog().size() > 0, timeout(seconds(90)));
+
- // ensure that a value was persisted in HBase
- List<Put> puts = profilerTable.getPutLog();
- assertEquals(1, puts.size());
-
- // generate the expected row key. only the profile name, entity, and period are used to generate the row key
- ProfileMeasurement measurement = new ProfileMeasurement()
- .withProfileName("profile-with-stats")
- .withEntity("global")
- .withPeriod(startAt, periodDurationMillis, TimeUnit.MILLISECONDS);
- RowKeyBuilder rowKeyBuilder = new SaltyRowKeyBuilder(saltDivisor, periodDurationMillis, TimeUnit.MILLISECONDS);
- byte[] expectedRowKey = rowKeyBuilder.rowKey(measurement);
-
- // ensure the correct row key was generated
- byte[] actualRowKey = puts.get(0).getRow();
- assertArrayEquals(failMessage(expectedRowKey, actualRowKey), expectedRowKey, actualRowKey);
++ // validate the measurements written by the batch profiler using `PROFILE_GET`
++ // the 'window' looks up to 5 hours before the max timestamp contained in the test data
++ assign("maxTimestamp", "1530978728982L");
++ assign("window", "PROFILE_WINDOW('from 5 hours ago', maxTimestamp)");
++
++ // retrieve the stats stored by the profiler
++ List results = execute("PROFILE_GET('profile-with-stats', 'global', window)", List.class);
++ assertTrue(results.size() > 0);
++ assertTrue(results.get(0) instanceof OnlineStatisticsProvider);
+ }
+
+ /**
+ * Generates an error message for if the byte comparison fails.
+ *
+ * @param expected The expected value.
+ * @param actual The actual value.
+ * @return
+ * @throws UnsupportedEncodingException
+ */
+ private String failMessage(byte[] expected, byte[] actual) throws UnsupportedEncodingException {
+ return String.format("expected '%s', got '%s'",
+ new String(expected, "UTF-8"),
+ new String(actual, "UTF-8"));
+ }
+
- /**
- * Generates the expected row key.
- *
- * @param profileName The name of the profile.
- * @param entity The entity.
- * @param whenMillis A timestamp in epoch milliseconds.
- * @return A row key.
- */
- private byte[] generateExpectedRowKey(String profileName, String entity, long whenMillis) {
-
- // only the profile name, entity, and period are used to generate the row key
- ProfileMeasurement measurement = new ProfileMeasurement()
- .withProfileName(profileName)
- .withEntity(entity)
- .withPeriod(whenMillis, periodDurationMillis, TimeUnit.MILLISECONDS);
-
- // build the row key
- RowKeyBuilder rowKeyBuilder = new SaltyRowKeyBuilder(saltDivisor, periodDurationMillis, TimeUnit.MILLISECONDS);
- return rowKeyBuilder.rowKey(measurement);
- }
-
- /**
- * Reads a value written by the Profiler.
- *
- * @param family The column family.
- * @param qualifier The column qualifier.
- * @param clazz The expected type of the value.
- * @param <T> The expected type of the value.
- * @return The value written by the Profiler.
- */
- private <T> List<T> read(List<Put> puts, String family, byte[] qualifier, Class<T> clazz) {
- List<T> results = new ArrayList<>();
-
- for(Put put: puts) {
- List<Cell> cells = put.get(Bytes.toBytes(family), qualifier);
- for(Cell cell : cells) {
- T value = SerDeUtils.fromBytes(cell.getValue(), clazz);
- results.add(value);
- }
- }
-
- return results;
++ private static String getMessage(String ipSource, long timestamp) {
++ return new MessageBuilder()
++ .withField("ip_src_addr", ipSource)
++ .withField("timestamp", timestamp)
++ .build()
++ .toJSONString();
+ }
+
+ @BeforeClass
+ public static void setupBeforeClass() throws UnableToStartException {
+
+ // create some messages that contain a timestamp - a really old timestamp; close to 1970
- message1 = new MessageBuilder()
- .withField("ip_src_addr", entity)
- .withField("timestamp", startAt)
- .build()
- .toJSONString();
-
- message2 = new MessageBuilder()
- .withField("ip_src_addr", entity)
- .withField("timestamp", startAt + 100)
- .build()
- .toJSONString();
-
- message3 = new MessageBuilder()
- .withField("ip_src_addr", entity)
- .withField("timestamp", startAt + (windowDurationMillis * 2))
- .build()
- .toJSONString();
-
- columnBuilder = new ValueOnlyColumnBuilder(columnFamily);
++ message1 = getMessage(entity, startAt);
++ message2 = getMessage(entity, startAt + 100);
++ message3 = getMessage(entity, startAt + (windowDurationMillis * 2));
+
+ // storm topology properties
+ final Properties topologyProperties = new Properties() {{
+
+ // storm settings
+ setProperty("profiler.workers", "1");
+ setProperty("profiler.executors", "0");
++
+ setProperty(Config.TOPOLOGY_AUTO_CREDENTIALS, "[]");
+ setProperty(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, "60");
+ setProperty(Config.TOPOLOGY_MAX_SPOUT_PENDING, "100000");
+
+ // ensure tuples are serialized during the test, otherwise serialization problems
+ // will not be found until the topology is run on a cluster with multiple workers
+ setProperty(Config.TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE, "true");
+ setProperty(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION, "false");
+ setProperty(Config.TOPOLOGY_KRYO_REGISTER, kryoSerializers);
+
+ // kafka settings
+ setProperty("profiler.input.topic", inputTopic);
+ setProperty("profiler.output.topic", outputTopic);
- setProperty("kafka.start", "UNCOMMITTED_EARLIEST");
++ setProperty("kafka.start", "EARLIEST");
+ setProperty("kafka.security.protocol", "PLAINTEXT");
+
+ // hbase settings
+ setProperty("profiler.hbase.salt.divisor", Integer.toString(saltDivisor));
+ setProperty("profiler.hbase.table", tableName);
+ setProperty("profiler.hbase.column.family", columnFamily);
+ setProperty("profiler.hbase.batch", "10");
+ setProperty("profiler.hbase.flush.interval.seconds", "1");
+ setProperty("hbase.provider.impl", "" + MockHBaseTableProvider.class.getName());
+
+ // profile settings
+ setProperty("profiler.period.duration", Long.toString(periodDurationMillis));
+ setProperty("profiler.period.duration.units", "MILLISECONDS");
+ setProperty("profiler.ttl", Long.toString(profileTimeToLiveMillis));
+ setProperty("profiler.ttl.units", "MILLISECONDS");
+ setProperty("profiler.window.duration", Long.toString(windowDurationMillis));
+ setProperty("profiler.window.duration.units", "MILLISECONDS");
+ setProperty("profiler.window.lag", Long.toString(windowLagMillis));
+ setProperty("profiler.window.lag.units", "MILLISECONDS");
+ setProperty("profiler.max.routes.per.bolt", Long.toString(maxRoutesPerBolt));
+ }};
+
+ // create the mock table
+ profilerTable = (MockHTable) MockHBaseTableProvider.addToCache(tableName, columnFamily);
+
+ zkComponent = getZKServerComponent(topologyProperties);
+
+ // create the input and output topics
+ kafkaComponent = getKafkaComponent(topologyProperties, Arrays.asList(
+ new KafkaComponent.Topic(inputTopic, 1),
+ new KafkaComponent.Topic(outputTopic, 1)));
+
+ // upload profiler configuration to zookeeper
+ configUploadComponent = new ConfigUploadComponent()
+ .withTopologyProperties(topologyProperties);
+
+ // load flux definition for the profiler topology
+ fluxComponent = new FluxTopologyComponent.Builder()
+ .withTopologyLocation(new File(FLUX_PATH))
+ .withTopologyName("profiler")
+ .withTopologyProperties(topologyProperties)
+ .build();
+
+ // start all components
+ runner = new ComponentRunner.Builder()
+ .withComponent("zk",zkComponent)
+ .withComponent("kafka", kafkaComponent)
+ .withComponent("config", configUploadComponent)
+ .withComponent("storm", fluxComponent)
+ .withMillisecondsBetweenAttempts(15000)
+ .withNumRetries(10)
+ .withCustomShutdownOrder(new String[] {"storm","config","kafka","zk"})
+ .build();
+ runner.start();
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ MockHBaseTableProvider.clear();
+ if (runner != null) {
+ runner.stop();
+ }
+ }
+
+ @Before
+ public void setup() {
+ // create the mock table
+ profilerTable = (MockHTable) MockHBaseTableProvider.addToCache(tableName, columnFamily);
++
++ // global properties
++ Map<String, Object> global = new HashMap<String, Object>() {{
++ put(PROFILER_HBASE_TABLE.getKey(), tableName);
++ put(PROFILER_COLUMN_FAMILY.getKey(), columnFamily);
++ put(PROFILER_HBASE_TABLE_PROVIDER.getKey(), MockHBaseTableProvider.class.getName());
++
++ // client needs to use the same period duration
++ put(PROFILER_PERIOD.getKey(), Long.toString(periodDurationMillis));
++ put(PROFILER_PERIOD_UNITS.getKey(), "MILLISECONDS");
++
++ // client needs to use the same salt divisor
++ put(PROFILER_SALT_DIVISOR.getKey(), saltDivisor);
++ }};
++
++ // create the stellar execution environment
++ executor = new DefaultStellarStatefulExecutor(
++ new SimpleFunctionResolver()
++ .withClass(GetProfile.class)
++ .withClass(FixedLookback.class)
++ .withClass(WindowLookback.class),
++ new Context.Builder()
++ .with(Context.Capabilities.GLOBAL_CONFIG, () -> global)
++ .build());
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ MockHBaseTableProvider.clear();
+ profilerTable.clear();
+ if (runner != null) {
+ runner.reset();
+ }
+ }
+
+ /**
+ * Uploads config values to Zookeeper.
+ * @param path The path on the local filesystem to the config values.
+ * @throws Exception
+ */
- public void uploadConfig(String path) throws Exception {
++ public void uploadConfigToZookeeper(String path) throws Exception {
+ configUploadComponent
+ .withGlobalConfiguration(path)
+ .withProfilerConfiguration(path)
+ .update();
+ }
++
++ /**
++ * Assign a value to the result of an expression.
++ *
++ * @param var The variable to assign.
++ * @param expression The expression to execute.
++ */
++ private void assign(String var, String expression) {
++ executor.assign(var, expression, Collections.emptyMap());
++ }
++
++ /**
++ * Execute a Stellar expression.
++ *
++ * @param expression The Stellar expression to execute.
++ * @param clazz
++ * @param <T>
++ * @return The result of executing the Stellar expression.
++ */
++ private <T> T execute(String expression, Class<T> clazz) {
++ T results = executor.execute(expression, Collections.emptyMap(), clazz);
++
++ LOG.debug("{} = {}", expression, results);
++ return results;
++ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/cad2f408/metron-analytics/metron-profiler-storm/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --cc metron-analytics/metron-profiler-storm/src/test/resources/log4j.properties
index 541f368,0000000..1c2359a
mode 100644,000000..100644
--- a/metron-analytics/metron-profiler-storm/src/test/resources/log4j.properties
+++ b/metron-analytics/metron-profiler-storm/src/test/resources/log4j.properties
@@@ -1,34 -1,0 +1,32 @@@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+# Root logger option
+log4j.rootLogger=ERROR, stdout
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
- log4j.appender.stdout.filter.1=org.apache.log4j.varia.StringMatchFilter
- log4j.appender.stdout.filter.1.StringToMatch=Connection timed out
- log4j.appender.stdout.filter.1.AcceptOnMatch=false
- log4j.appender.stdout.filter.2=org.apache.log4j.varia.StringMatchFilter
- log4j.appender.stdout.filter.2.StringToMatch=Background
- log4j.appender.stdout.filter.2.AcceptOnMatch=false
++
++# uncomment below to help debug tests
++#log4j.logger.org.apache.metron.profiler=ALL
++#log4j.logger.org.apache.storm.windowing=ALL
[21/51] [abbrv] metron git commit: METRON-1798 Add mpack support for
parser aggregation (anandsubbu) closes apache/metron#1215
Posted by ot...@apache.org.
METRON-1798 Add mpack support for parser aggregation (anandsubbu) closes apache/metron#1215
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ebdaf5f9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ebdaf5f9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ebdaf5f9
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: ebdaf5f905829d8ff59814bb3555666217f85abb
Parents: ff1f9cf
Author: anandsubbu <an...@gmail.com>
Authored: Wed Oct 3 18:57:48 2018 +0530
Committer: anandsubbu <an...@apache.org>
Committed: Wed Oct 3 18:57:48 2018 +0530
----------------------------------------------------------------------
.../configuration/metron-parsers-env.xml | 2 +-
.../CURRENT/package/scripts/parser_commands.py | 49 ++++++++++++++++++--
.../metron-parsers/ParserChaining.md | 18 +++++++
3 files changed, 63 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/ebdaf5f9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
index a9a498b..03a2594 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
@@ -21,7 +21,7 @@
<property>
<name>parsers</name>
<value>bro,snort,yaf</value>
- <description>Metron parsers to deploy</description>
+ <description>Metron parsers to deploy. You can also specify an aggregated parser list by grouping them with double quotes. For example: "parserA,parserB",parserC,parserD</description>
<display-name>Metron Parsers</display-name>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/metron/blob/ebdaf5f9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
index 274306a..18780d9 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
@@ -20,6 +20,7 @@ limitations under the License.
import os
import re
+import shlex
import subprocess
import time
@@ -49,7 +50,45 @@ class ParserCommands:
# get list of parsers
def __get_parsers(self, params):
- return params.parsers.replace(' ', '').split(',')
+ """
+ Combines the list of parser topics and sends a unique list to be used for
+ Kafka topic creation and the like.
+ :param params:
+ :return: List containing the names of unique parsers
+ """
+ parserBatches = list(self.__get_aggr_parsers(params))
+ parsers = ','.join(s.translate(None, '"') for s in parserBatches)
+ # Get only the unique list of parser names
+ parsers = list(set(parsers.split(',')))
+ return parsers
+
+ def __get_aggr_parsers(self, params):
+ """
+ Fetches the list of aggregated (and regular) parsers and returns a list.
+ If the input list of parsers were "bro,snort,yaf", "bro,snort" and yaf, for example,
+ then this method will return ["bro,snort,yaf", "bro,snort", "yaf"]
+ :param params:
+ :return: List containing the names of parsers
+ """
+ parserList = []
+ parsers = shlex.shlex(params.parsers)
+ for name in parsers:
+ parserList.append(name.strip(','))
+ return [s.translate(None, "'[]") for s in filter(None, parserList)]
+
+ def get_parser_aggr_topology_names(self, params):
+ """
+ Returns the names of regular and aggregated topologies as they would run in storm
+ An aggregated topology has the naming convention of 'parserA__parserB'.
+ For example, a list of parsers like ["bro,snort", yaf] will be returned as ["bro__snort", "yaf"]
+ :param params:
+ :return: List containing the names of parser topologies
+ """
+ topologyName = []
+ for parser in self.__get_aggr_parsers(params):
+ parser = parser.replace(",", "__").strip('"')
+ topologyName.append(parser)
+ return topologyName
def __get_topics(self):
# All errors go to indexing topics, so create it here if it's not already
@@ -104,7 +143,7 @@ class ParserCommands:
metron_service.init_kafka_acl_groups(self.__params, self.__get_kafka_acl_groups())
def start_parser_topologies(self, env):
- Logger.info("Starting Metron parser topologies: {0}".format(self.get_parser_list()))
+ Logger.info("Starting Metron parser topologies: {0}".format(self.__get_aggr_parsers(self.__params)))
start_cmd_template = """{0}/bin/start_parser_topology.sh \
-k {1} \
-z {2} \
@@ -118,7 +157,7 @@ class ParserCommands:
self.__params.metron_principal_name,
execute_user=self.__params.metron_user)
- stopped_parsers = set(self.get_parser_list()) - self.get_running_topology_names(env)
+ stopped_parsers = set(self.__get_aggr_parsers(self.__params)) - self.get_running_topology_names(env)
Logger.info('Parsers that need started: ' + str(stopped_parsers))
for parser in stopped_parsers:
@@ -135,7 +174,7 @@ class ParserCommands:
def stop_parser_topologies(self, env):
Logger.info('Stopping parsers')
- running_parsers = set(self.get_parser_list()) & self.get_running_topology_names(env)
+ running_parsers = set(self.get_parser_aggr_topology_names(self.__params)) & self.get_running_topology_names(env)
Logger.info('Parsers that need stopped: ' + str(running_parsers))
for parser in running_parsers:
@@ -192,7 +231,7 @@ class ParserCommands:
env.set_params(self.__params)
all_running = True
topologies = metron_service.get_running_topologies(self.__params)
- for parser in self.get_parser_list():
+ for parser in self.get_parser_aggr_topology_names(self.__params):
parser_found = False
is_running = False
if parser in topologies:
http://git-wip-us.apache.org/repos/asf/metron/blob/ebdaf5f9/metron-platform/metron-parsers/ParserChaining.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/ParserChaining.md b/metron-platform/metron-parsers/ParserChaining.md
index a28f11d..4cab74c 100644
--- a/metron-platform/metron-parsers/ParserChaining.md
+++ b/metron-platform/metron-parsers/ParserChaining.md
@@ -49,6 +49,24 @@ data emitted from any parser, we will need to adjust the downstream
parsers to extract the enveloped data from the JSON blob and treat it as
the data to parse.
+# Aggregated Parsers with Parser Chaining
+Chained parsers can be run as aggregated parsers. These parsers continue to use the sensor specific Kafka topics, and do not do internal routing to the appropriate sensor.
+
+Say, there were three sensors (`bro`, `snort` and `yaf`). Instead of creating a topology per sensor, all 3 can be run in a single aggregated parser. It is also possible to aggregate a subset of these parsers (e.g. run `bro` as it's own topology, and aggregate the other 2).
+
+The step to start an aggregated parsers then becomes
+```
+$METRON_HOME/bin/start_parser_topology.sh -k $BROKERLIST -z $ZOOKEEPER -s bro,snort,yaf
+```
+
+which will result in a single storm topology named `bro__snort__yaf` to run.
+
+Aggregated parsers can be specified using the Ambari Metron config as well under Services -> Metron -> Configs -> 'Parsers' tab -> 'Metron Parsers' field. The grouping is configured by enclosing the desired parsers in double quotes.
+
+Some examples of specifying aggregated parsers are as follows:
+* "bro,snort,yaf" --> Will start a single topology named `bro__snort__yaf`
+* "ciscopixA,ciscopixB",yaf,"squid,ciscopixC" --> Will start three topologies viz. `ciscopixA__ciscopixB`, `yaf` and `squid__ciscopixC`
+
# Architecting a Parser Chaining Solution in Metron
Currently the approach to fulfill this requirement involves a couple
[23/51] [abbrv] metron git commit: METRON-1791 Add GUID to Messages
Produced by Profiler (nickwallen) closes apache/metron#1210
Posted by ot...@apache.org.
METRON-1791 Add GUID to Messages Produced by Profiler (nickwallen) closes apache/metron#1210
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7e222fa4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7e222fa4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7e222fa4
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 7e222fa47aedc4ab3bafec54590ec0bc73b5f75c
Parents: b872fdc
Author: nickwallen <ni...@nickallen.org>
Authored: Thu Oct 4 17:45:32 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Thu Oct 4 17:45:32 2018 -0400
----------------------------------------------------------------------
.../metron/profiler/storm/KafkaEmitter.java | 31 +++++++++---
.../zookeeper/triage-result/profiler.json | 20 ++++++++
.../metron/profiler/storm/KafkaEmitterTest.java | 4 ++
.../integration/ProfilerIntegrationTest.java | 52 ++++++++++++++------
4 files changed, 85 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/7e222fa4/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java
index af1fbca..adbde1b 100644
--- a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java
+++ b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/KafkaEmitter.java
@@ -21,6 +21,7 @@ package org.apache.metron.profiler.storm;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.ClassUtils;
+import org.apache.metron.common.Constants;
import org.apache.metron.profiler.ProfileMeasurement;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
@@ -33,6 +34,7 @@ import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.lang.invoke.MethodHandles;
import java.util.Map;
+import java.util.UUID;
/**
* Responsible for emitting a {@link ProfileMeasurement} to an output stream that will
@@ -42,6 +44,14 @@ public class KafkaEmitter implements ProfileMeasurementEmitter, Serializable {
protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String PROFILE_FIELD = "profile";
+ public static final String ENTITY_FIELD = "entity";
+ public static final String PERIOD_ID_FIELD = "period";
+ public static final String PERIOD_START_FIELD = "period.start";
+ public static final String PERIOD_END_FIELD = "period.end";
+ public static final String TIMESTAMP_FIELD = "timestamp";
+ public static final String ALERT_FIELD = "is_alert";
+
/**
* The stream identifier used for this destination;
*/
@@ -126,14 +136,15 @@ public class KafkaEmitter implements ProfileMeasurementEmitter, Serializable {
private JSONObject createMessage(ProfileMeasurement measurement) {
JSONObject message = new JSONObject();
- message.put("profile", measurement.getDefinition().getProfile());
- message.put("entity", measurement.getEntity());
- message.put("period", measurement.getPeriod().getPeriod());
- message.put("period.start", measurement.getPeriod().getStartTimeMillis());
- message.put("period.end", measurement.getPeriod().getEndTimeMillis());
- message.put("timestamp", System.currentTimeMillis());
- message.put("source.type", sourceType);
- message.put("is_alert", "true");
+ message.put(PROFILE_FIELD, measurement.getDefinition().getProfile());
+ message.put(ENTITY_FIELD, measurement.getEntity());
+ message.put(PERIOD_ID_FIELD, measurement.getPeriod().getPeriod());
+ message.put(PERIOD_START_FIELD, measurement.getPeriod().getStartTimeMillis());
+ message.put(PERIOD_END_FIELD, measurement.getPeriod().getEndTimeMillis());
+ message.put(TIMESTAMP_FIELD, System.currentTimeMillis());
+ message.put(Constants.SENSOR_TYPE, sourceType);
+ message.put(ALERT_FIELD, "true");
+ message.put(Constants.GUID, UUID.randomUUID().toString());
return message;
}
@@ -158,6 +169,10 @@ public class KafkaEmitter implements ProfileMeasurementEmitter, Serializable {
this.streamId = streamId;
}
+ public String getSourceType() {
+ return sourceType;
+ }
+
public void setSourceType(String sourceType) {
this.sourceType = sourceType;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/7e222fa4/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/triage-result/profiler.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/triage-result/profiler.json b/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/triage-result/profiler.json
new file mode 100644
index 0000000..7d63da7
--- /dev/null
+++ b/metron-analytics/metron-profiler-storm/src/test/config/zookeeper/triage-result/profiler.json
@@ -0,0 +1,20 @@
+{
+ "profiles": [
+ {
+ "profile": "profile-with-triage",
+ "foreach": "'global'",
+ "update": {
+ "stats": "STATS_ADD(stats, 1)"
+ },
+ "result": {
+ "profile": "stats",
+ "triage": {
+ "min": "STATS_MIN(stats)",
+ "max": "STATS_MAX(stats)",
+ "mean": "STATS_MEAN(stats)"
+ }
+ }
+ }
+ ],
+ "timestampField": "timestamp"
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/7e222fa4/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/KafkaEmitterTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/KafkaEmitterTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/KafkaEmitterTest.java
index 51ca3a4..86849ac 100644
--- a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/KafkaEmitterTest.java
+++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/KafkaEmitterTest.java
@@ -22,6 +22,7 @@ package org.apache.metron.profiler.storm;
import com.google.common.collect.ImmutableMap;
import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.profiler.ProfileConfig;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.profiler.ProfileMeasurement;
@@ -148,6 +149,7 @@ public class KafkaEmitterTest {
assertEquals(measurement.getPeriod().getEndTimeMillis(), actual.get("period.end"));
assertEquals("profiler", actual.get("source.type"));
assertNotNull(actual.get("timestamp"));
+ assertNotNull(actual.get(Constants.GUID));
// validate that the triage value has been added
assertEquals(measurement.getTriageValues().get("triage-key"), actual.get("triage-key"));
@@ -214,6 +216,8 @@ public class KafkaEmitterTest {
assertEquals(measurement.getPeriod().getStartTimeMillis(), actual.get("period.start"));
assertEquals(measurement.getPeriod().getEndTimeMillis(), actual.get("period.end"));
assertEquals("profiler", actual.get("source.type"));
+ assertNotNull(actual.get("timestamp"));
+ assertNotNull(actual.get(Constants.GUID));
// the invalid expression should be skipped and not included in the message
assertFalse(actual.containsKey("invalid"));
http://git-wip-us.apache.org/repos/asf/metron/blob/7e222fa4/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
index 4389d42..64a1482 100644
--- a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
+++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/integration/ProfilerIntegrationTest.java
@@ -40,6 +40,8 @@ import org.apache.metron.stellar.common.StellarStatefulExecutor;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver;
import org.apache.storm.Config;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -49,7 +51,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.UnsupportedEncodingException;
import java.lang.invoke.MethodHandles;
import java.util.Arrays;
import java.util.Collections;
@@ -68,6 +69,13 @@ import static org.apache.metron.profiler.client.stellar.ProfilerClientConfig.PRO
import static org.apache.metron.profiler.client.stellar.ProfilerClientConfig.PROFILER_PERIOD;
import static org.apache.metron.profiler.client.stellar.ProfilerClientConfig.PROFILER_PERIOD_UNITS;
import static org.apache.metron.profiler.client.stellar.ProfilerClientConfig.PROFILER_SALT_DIVISOR;
+import static org.apache.metron.profiler.storm.KafkaEmitter.ALERT_FIELD;
+import static org.apache.metron.profiler.storm.KafkaEmitter.ENTITY_FIELD;
+import static org.apache.metron.profiler.storm.KafkaEmitter.PERIOD_END_FIELD;
+import static org.apache.metron.profiler.storm.KafkaEmitter.PERIOD_ID_FIELD;
+import static org.apache.metron.profiler.storm.KafkaEmitter.PERIOD_START_FIELD;
+import static org.apache.metron.profiler.storm.KafkaEmitter.PROFILE_FIELD;
+import static org.apache.metron.profiler.storm.KafkaEmitter.TIMESTAMP_FIELD;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -285,18 +293,35 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
assertTrue(results.get(0) instanceof OnlineStatisticsProvider);
}
- /**
- * Generates an error message for if the byte comparison fails.
- *
- * @param expected The expected value.
- * @param actual The actual value.
- * @return
- * @throws UnsupportedEncodingException
- */
- private String failMessage(byte[] expected, byte[] actual) throws UnsupportedEncodingException {
- return String.format("expected '%s', got '%s'",
- new String(expected, "UTF-8"),
- new String(actual, "UTF-8"));
+ @Test
+ public void testProfileWithTriageResult() throws Exception {
+ uploadConfigToZookeeper(TEST_RESOURCES + "/config/zookeeper/triage-result");
+
+ // start the topology and write test messages to kafka
+ fluxComponent.submitTopology();
+ List<String> telemetry = FileUtils.readLines(new File("src/test/resources/telemetry.json"));
+ kafkaComponent.writeMessages(inputTopic, telemetry);
+
+ // wait until the triage message is output to kafka
+ waitOrTimeout(() -> kafkaComponent.readMessages(outputTopic).size() > 0, timeout(seconds(90)));
+
+ List<byte[]> outputMessages = kafkaComponent.readMessages(outputTopic);
+ assertEquals(1, outputMessages.size());
+
+ // validate the triage message
+ JSONObject message = (JSONObject) new JSONParser().parse(new String(outputMessages.get(0), "UTF-8"));
+ assertEquals("profile-with-triage", message.get(PROFILE_FIELD));
+ assertEquals("global", message.get(ENTITY_FIELD));
+ assertEquals(76548935L, message.get(PERIOD_ID_FIELD));
+ assertEquals(1530978700000L, message.get(PERIOD_START_FIELD));
+ assertEquals(1530978720000L, message.get(PERIOD_END_FIELD));
+ assertEquals("profiler", message.get(Constants.SENSOR_TYPE));
+ assertEquals("true", message.get(ALERT_FIELD));
+ assertEquals(1.0, message.get("min"));
+ assertEquals(1.0, message.get("max"));
+ assertEquals(1.0, message.get("mean"));
+ assertTrue(message.containsKey(TIMESTAMP_FIELD));
+ assertTrue(message.containsKey(Constants.GUID));
}
private static String getMessage(String ipSource, long timestamp) {
@@ -471,7 +496,6 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
*/
private <T> T execute(String expression, Class<T> clazz) {
T results = executor.execute(expression, Collections.emptyMap(), clazz);
-
LOG.debug("{} = {}", expression, results);
return results;
}
[31/51] [abbrv] metron git commit: METRON-1806: Upgrade Maven Shade
Plugin version (mmiklavc via mmiklavc) closes apache/metron#1224
Posted by ot...@apache.org.
METRON-1806: Upgrade Maven Shade Plugin version (mmiklavc via mmiklavc) closes apache/metron#1224
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e4823667
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e4823667
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e4823667
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: e482366726b07173fbc9ff9b084b851596e13005
Parents: 5bfc08c
Author: mmiklavc <mi...@gmail.com>
Authored: Tue Oct 9 09:22:22 2018 -0600
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Tue Oct 9 09:22:22 2018 -0600
----------------------------------------------------------------------
.gitignore | 1 +
pom.xml | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/e4823667/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index a0b9691..df3f030 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,7 @@ target/
*.tmp
*.bak
*.class
+*.pyc
tmp/**
tmp/**/*
http://git-wip-us.apache.org/repos/asf/metron/blob/e4823667/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a98de72..1e6adb0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,7 +126,7 @@
<global_solr_version>6.6.2</global_solr_version>
<global_mockito_version>1.10.19</global_mockito_version>
<global_powermock_version>1.7.0</global_powermock_version>
- <global_shade_version>2.4.3</global_shade_version>
+ <global_shade_version>3.2.0</global_shade_version>
<global_jackson_version>2.7.4</global_jackson_version>
<global_errorprone_core_version>2.0.14</global_errorprone_core_version>
<global_jar_version>3.0.2</global_jar_version>
[06/51] [abbrv] metron git commit: METRON-1760 Kill PCAP job should
prompt for confirmation (ruffle via nickwallen) closes apache/metron#1199
Posted by ot...@apache.org.
METRON-1760 Kill PCAP job should prompt for confirmation (ruffle via nickwallen) closes apache/metron#1199
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/91c410fd
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/91c410fd
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/91c410fd
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 91c410fd2016d8776aba650b41239c971b706d01
Parents: 8b2f297
Author: ruffle <ft...@gmail.com>
Authored: Tue Sep 18 11:11:44 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Tue Sep 18 11:11:44 2018 -0400
----------------------------------------------------------------------
.../metron-alerts/package-lock.json | 14 +++
metron-interface/metron-alerts/package.json | 1 +
.../pcap/pcap-panel/pcap-panel.component.html | 17 +++-
.../pcap-panel/pcap-panel.component.spec.ts | 36 +++++++-
.../app/pcap/pcap-panel/pcap-panel.component.ts | 1 +
.../metron-alerts/src/app/pcap/pcap.module.ts | 4 +-
.../metron-alerts/src/confirm-popover.scss | 95 ++++++++++++++++++++
metron-interface/metron-alerts/src/styles.scss | 1 +
8 files changed, 166 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/91c410fd/metron-interface/metron-alerts/package-lock.json
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/package-lock.json b/metron-interface/metron-alerts/package-lock.json
index 7d25b51..f927ffe 100644
--- a/metron-interface/metron-alerts/package-lock.json
+++ b/metron-interface/metron-alerts/package-lock.json
@@ -5808,6 +5808,15 @@
"integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=",
"dev": true
},
+ "angular-confirmation-popover": {
+ "version": "4.2.0",
+ "resolved": "https://registry.npmjs.org/angular-confirmation-popover/-/angular-confirmation-popover-4.2.0.tgz",
+ "integrity": "sha512-ItCPzV52user93NRk9rF4Rp8NpawBWJdkNf8+6lH//f5i/N5HY0Aq5Hcch3xk19h9P48k0WZnfwOQL181xe4MQ==",
+ "requires": {
+ "positioning": "^1.3.1",
+ "tslib": "^1.9.0"
+ }
+ },
"ansi-html": {
"version": "0.0.7",
"resolved": "https://registry.npmjs.org/ansi-html/-/ansi-html-0.0.7.tgz",
@@ -14148,6 +14157,11 @@
}
}
},
+ "positioning": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/positioning/-/positioning-1.4.0.tgz",
+ "integrity": "sha512-LbN+mgAXtcDdN46xMJ3yZwjndqqYJODaO5qKmU+MVMu5tL3K2dlm1Qha/zh1k2JAFym5HDaZpnPfO4gr91VTRw=="
+ },
"posix-character-classes": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz",
http://git-wip-us.apache.org/repos/asf/metron/blob/91c410fd/metron-interface/metron-alerts/package.json
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/package.json b/metron-interface/metron-alerts/package.json
index 2133b74..14172fd 100644
--- a/metron-interface/metron-alerts/package.json
+++ b/metron-interface/metron-alerts/package.json
@@ -26,6 +26,7 @@
"@types/jquery": "^3.3.4",
"ace-builds": "^1.2.6",
"ajv": "^6.5.1",
+ "angular-confirmation-popover": "^4.2.0",
"bootstrap": "4.0.0-alpha.6",
"core-js": "^2.4.1",
"font-awesome": "^4.7.0",
http://git-wip-us.apache.org/repos/asf/metron/blob/91c410fd/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
index 6b18a5f..37e21ee 100644
--- a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
+++ b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
@@ -19,7 +19,22 @@
<div class="progress pcap-progress-background">
<div class="progress-bar progress-bar-animated pcap-progress" role="progressbar" attr.aria-valuenow="{{progressWidth}}" aria-valuemin="0" aria-valuemax="100" [ngStyle]="{'width': progressWidth + '%'}">{{progressWidth}}%</div>
</div>
- <button data-qe-id="pcap-cancel-query-button" class="pcap-cancel-query-button btn btn-primary btn-sm" (click)="cancelQuery()" [disabled]="!queryId"></button>
+ <button
+ mwlConfirmationPopover
+ [popoverMessage]="cancelConfirmMessage"
+ placement="left"
+ (confirm)="cancelQuery()"
+ cancelText="No"
+ confirmText="Yes"
+ confirmButtonType="danger"
+ appendToBody="true"
+ popoverClass="pcap-cancel-query-confirm-popover confirm-popover"
+
+ data-qe-id="pcap-cancel-query-button"
+ class="pcap-cancel-query-button btn btn-primary btn-sm"
+ [disabled]="!queryId"
+ >
+ </button>
</div>
<div *ngIf="errorMsg" class="alert alert-danger" role="alert" data-qe-id="error">
{{ errorMsg }}
http://git-wip-us.apache.org/repos/asf/metron/blob/91c410fd/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
index aa9b799..29bd649 100644
--- a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
@@ -27,6 +27,7 @@ import { By } from '../../../../node_modules/@angular/platform-browser';
import { PcapRequest } from '../model/pcap.request';
import { of, defer } from 'rxjs';
import { RestError } from '../../model/rest-error';
+import { ConfirmationPopoverModule } from 'angular-confirmation-popover';
@Component({
selector: 'app-pcap-filters',
@@ -66,6 +67,9 @@ describe('PcapPanelComponent', () => {
beforeEach(async(() => {
TestBed.configureTestingModule({
+ imports: [
+ ConfirmationPopoverModule.forRoot(),
+ ],
declarations: [
FakeFilterComponent,
FakePcapListComponent,
@@ -346,7 +350,7 @@ describe('PcapPanelComponent', () => {
expect(fixture.debugElement.query(By.css('[data-qe-id="pcap-cancel-query-button"]'))).toBeDefined();
});
- it('should hide the progress bar if the user clicks on the cancel button', fakeAsync(() => {
+ it('should hide the progress bar if the user clicks on the cancel button and confirms', fakeAsync(() => {
component.queryRunning = true;
component.queryId = '42';
fixture.detectChanges();
@@ -356,6 +360,12 @@ describe('PcapPanelComponent', () => {
const cancelBtnEl = cancelBtn.nativeElement;
cancelBtnEl.click();
+ fixture.detectChanges();
+
+ const confirmButton = fixture.debugElement.query(By.css('.pcap-cancel-query-confirm-popover .btn-danger'));
+ const confirmButtonEl = confirmButton.nativeElement;
+
+ confirmButtonEl.click();
tick();
fixture.detectChanges();
@@ -377,6 +387,12 @@ describe('PcapPanelComponent', () => {
const cancelBtnEl = cancelBtn.nativeElement;
cancelBtnEl.click();
+ fixture.detectChanges();
+
+ const confirmButton = fixture.debugElement.query(By.css('.pcap-cancel-query-confirm-popover .btn-danger'));
+ const confirmButtonEl = confirmButton.nativeElement;
+
+ confirmButtonEl.click();
tick();
fixture.detectChanges();
@@ -399,6 +415,12 @@ describe('PcapPanelComponent', () => {
const cancelBtnEl = cancelBtn.nativeElement;
cancelBtnEl.click();
+ fixture.detectChanges();
+
+ const confirmButton = fixture.debugElement.query(By.css('.pcap-cancel-query-confirm-popover .btn-danger'));
+ const confirmButtonEl = confirmButton.nativeElement;
+
+ confirmButtonEl.click();
tick();
fixture.detectChanges();
@@ -438,6 +460,12 @@ describe('PcapPanelComponent', () => {
const cancelBtnEl = cancelBtn.nativeElement;
cancelBtnEl.click();
+ fixture.detectChanges();
+
+ const confirmButton = fixture.debugElement.query(By.css('.pcap-cancel-query-confirm-popover .btn-danger'));
+ const confirmButtonEl = confirmButton.nativeElement;
+
+ confirmButtonEl.click();
tick();
fixture.detectChanges();
@@ -457,6 +485,12 @@ describe('PcapPanelComponent', () => {
const cancelBtnEl = cancelBtn.nativeElement;
cancelBtnEl.click();
+ fixture.detectChanges();
+
+ const confirmButton = fixture.debugElement.query(By.css('.pcap-cancel-query-confirm-popover .btn-danger'));
+ const confirmButtonEl = confirmButton.nativeElement;
+
+ confirmButtonEl.click();
tick();
fixture.detectChanges();
http://git-wip-us.apache.org/repos/asf/metron/blob/91c410fd/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
index 5f262e9..8e4c375 100644
--- a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
@@ -46,6 +46,7 @@ export class PcapPanelComponent implements OnInit, OnDestroy {
pagination: PcapPagination = new PcapPagination();
savedPcapRequest: {};
errorMsg: string;
+ cancelConfirmMessage = 'Are you sure want to cancel the running query?';
constructor(private pcapService: PcapService) { }
http://git-wip-us.apache.org/repos/asf/metron/blob/91c410fd/metron-interface/metron-alerts/src/app/pcap/pcap.module.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/pcap/pcap.module.ts b/metron-interface/metron-alerts/src/app/pcap/pcap.module.ts
index a789a8a..be61438 100644
--- a/metron-interface/metron-alerts/src/app/pcap/pcap.module.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/pcap.module.ts
@@ -29,6 +29,7 @@ import { PcapPanelComponent } from './pcap-panel/pcap-panel.component';
import { PcapPacketLineComponent } from './pcap-packet-line/pcap-packet-line.component';
import { PcapPaginationComponent } from './pcap-pagination/pcap-pagination.component';
import { PcapService } from './service/pcap.service';
+import { ConfirmationPopoverModule } from 'angular-confirmation-popover';
@NgModule({
imports: [
@@ -37,7 +38,8 @@ import { PcapService } from './service/pcap.service';
FormsModule,
HttpClientModule,
ReactiveFormsModule,
- DatePickerModule
+ DatePickerModule,
+ ConfirmationPopoverModule.forRoot()
],
declarations: [
PcapListComponent,
http://git-wip-us.apache.org/repos/asf/metron/blob/91c410fd/metron-interface/metron-alerts/src/confirm-popover.scss
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/confirm-popover.scss b/metron-interface/metron-alerts/src/confirm-popover.scss
new file mode 100644
index 0000000..2c05270
--- /dev/null
+++ b/metron-interface/metron-alerts/src/confirm-popover.scss
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+$popover-button-color-confirm: #006EA0;
+$popover-button-border-color-confirm: $popover-button-color-confirm;
+
+$popover-button-color-cancel: #333333;
+$popover-button-border-color-cancel: #0F6F9E;
+
+$popover-background-color: #0C3B43;
+$popover-border-color: #1B596C;
+
+.confirm-popover {
+ background-color: $popover-background-color;
+ border: 1px solid $popover-border-color;
+ color: #999999;
+
+ & .btn {
+ font-size: 14px;
+ cursor: pointer;
+ }
+
+ & .btn-default {
+ background-color: $popover-button-color-cancel;
+ border-color: $popover-button-border-color-cancel;
+ color: #32ABDF;
+
+ &:hover {
+ background-color: #004b6d;
+ }
+ }
+
+ & .btn-danger {
+ background-color: $popover-button-color-confirm;
+ border-color: $popover-button-border-color-confirm;
+ color: white;
+
+ &:hover {
+ background-color: #004b6d;
+ border-color: #004b6d;
+ }
+ }
+
+
+ &.popover-top {
+ &::before {
+ border-top-color: $popover-border-color;
+ }
+ &::after {
+ border-top-color: $popover-background-color;
+ bottom: -9px;
+ }
+ }
+ &.popover-right {
+ &::before {
+ border-right-color: $popover-border-color;
+ }
+ &::after {
+ border-right-color: $popover-background-color;
+ left: -9px;
+ }
+ }
+ &.popover-bottom {
+ &::before {
+ border-bottom-color: $popover-border-color;
+ }
+ &::after {
+ border-bottom-color: $popover-background-color;
+ top: -9px;
+ }
+ }
+ &.popover-left {
+ &::before {
+ border-left-color: $popover-border-color;
+ }
+ &::after {
+ border-left-color: $popover-background-color;
+ right: -9px;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/91c410fd/metron-interface/metron-alerts/src/styles.scss
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/styles.scss b/metron-interface/metron-alerts/src/styles.scss
index 687ad14..614d539 100644
--- a/metron-interface/metron-alerts/src/styles.scss
+++ b/metron-interface/metron-alerts/src/styles.scss
@@ -22,6 +22,7 @@
@import "metron-dialog.scss";
@import "../node_modules/pikaday-time/scss/pikaday.scss";
@import "hexagon";
+@import "confirm-popover.scss";
body,
button {
[11/51] [abbrv] metron git commit: METRON-1709 Add controls to start
/ stop the PCAP topology from Ambari. (MohanDV via nickwallen) closes
apache/metron#1201
Posted by ot...@apache.org.
METRON-1709 Add controls to start / stop the PCAP topology from Ambari. (MohanDV via nickwallen) closes apache/metron#1201
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/15194c3b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/15194c3b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/15194c3b
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 15194c3b4e7d9d9d161b0ddfa556395b34cfef7c
Parents: 3372aa3
Author: MohanDV <mo...@gmail.com>
Authored: Thu Sep 20 11:10:07 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Thu Sep 20 11:10:07 2018 -0400
----------------------------------------------------------------------
.../roles/ambari_config/vars/single_node_vm.yml | 3 +-
.../roles/ambari_config/vars/small_cluster.yml | 3 +-
.../METRON/CURRENT/role_command_order.json | 9 +-
.../common-services/METRON/CURRENT/metainfo.xml | 39 ++++
.../package/scripts/params/params_linux.py | 4 +
.../package/scripts/params/status_params.py | 1 +
.../CURRENT/package/scripts/pcap_commands.py | 201 +++++++++++++++++++
.../CURRENT/package/scripts/pcap_master.py | 105 ++++++++++
.../CURRENT/package/scripts/rest_commands.py | 43 ----
.../CURRENT/package/scripts/rest_master.py | 7 -
.../METRON/CURRENT/service_advisor.py | 5 +
11 files changed, 365 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index a3c643b..962726c 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -32,6 +32,7 @@ es_master: [ES_MASTER]
kibana_master: [KIBANA_MASTER]
metron_indexing: [METRON_INDEXING]
metron_profiler: [METRON_PROFILER]
+metron_pcap: [METRON_PCAP]
metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
metron_parsers : [METRON_PARSERS]
metron_rest: [METRON_REST]
@@ -40,7 +41,7 @@ metron_alerts_ui: [METRON_ALERTS_UI]
metron_components: >
{{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) |
- union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) | union(metron_indexing) | union(metron_profiler) |
+ union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) | union(metron_indexing) | union(metron_profiler) | union(metron_pcap) |
union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(es_master) }}
cluster_name: "metron_cluster"
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
index 218e267..818b5f3 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
@@ -32,6 +32,7 @@ es_slave: [ES_SLAVE]
kibana_master: [KIBANA_MASTER]
metron_indexing: [METRON_INDEXING]
metron_profiler: [METRON_PROFILER]
+metron_pcap: [METRON_PCAP]
metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
metron_parsers : [METRON_PARSERS]
metron_rest: [METRON_REST]
@@ -45,7 +46,7 @@ master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_ma
master_2_host:
- "{{groups.ambari_slave[1]}}"
metron_components: >
- {{ metron_indexing | union(metron_profiler) | union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(hadoop_slave) | union(storm_slave) |
+ {{ metron_indexing | union(metron_profiler) | union(metron_pcap) |union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(hadoop_slave) | union(storm_slave) |
union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}
metron_host:
- "{{ groups.metron[0] }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
index fcb2206..ee6ca23 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
@@ -6,23 +6,26 @@
"METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"],
"METRON_ENRICHMENT-INSTALL": ["METRON_INDEXING-INSTALL"],
"METRON_PROFILER-INSTALL" : ["METRON_ENRICHMENT-INSTALL"],
+ "METRON_PCAP-INSTALL" : ["METRON_INDEXING-INSTALL"],
"METRON_REST-INSTALL" : ["METRON_PARSERS-INSTALL"],
"METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START" ,"METRON_ENRICHMENT_MASTER-START"],
"METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
"METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "METRON_PARSERS-START"],
"METRON_PROFILER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START", "METRON_ENRICHMENT-INSTALL"],
+ "METRON_PCAP-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
"METRON_REST-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "METRON_PARSERS-INSTALL","METRON_INDEXING-INSTALL","METRON_ENRICHMENT-INSTALL"],
"METRON_MANAGEMENT_UI-START" : ["METRON_REST-START"],
"METRON_ALERTS_UI-START" : ["METRON_REST-START"],
- "STORM_REST_API-STOP" : ["METRON_PARSERS-STOP", "METRON_ENRICHMENT_MASTER-STOP", "METRON_INDEXING-STOP", "METRON_PROFILER-STOP", "METRON_REST-STOP", "METRON_MANAGEMENT_UI-STOP", "METRON_ALERTS_UI-STOP"],
- "STORM_UI_SERVER-STOP" : ["METRON_PARSERS-STOP", "METRON_ENRICHMENT_MASTER-STOP", "METRON_INDEXING-STOP", "METRON_PROFILER-STOP", "METRON_REST-STOP", "METRON_MANAGEMENT_UI-STOP", "METRON_ALERTS_UI-STOP"],
+ "STORM_REST_API-STOP" : ["METRON_PARSERS-STOP", "METRON_ENRICHMENT_MASTER-STOP", "METRON_INDEXING-STOP", "METRON_PROFILER-STOP", "METRON_PCAP-STOP", "METRON_REST-STOP", "METRON_MANAGEMENT_UI-STOP", "METRON_ALERTS_UI-STOP"],
+ "STORM_UI_SERVER-STOP" : ["METRON_PARSERS-STOP", "METRON_ENRICHMENT_MASTER-STOP", "METRON_INDEXING-STOP", "METRON_PROFILER-STOP", "METRON_PCAP-STOP", "METRON_REST-STOP", "METRON_MANAGEMENT_UI-STOP", "METRON_ALERTS_UI-STOP"],
- "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START", "METRON_ENRICHMENT_MASTER-START", "METRON_INDEXING-START", "METRON_PROFILER-START", "METRON_REST-START", "METRON_MANAGEMENT_UI-START", "METRON_ALERTS_UI-START"],
+ "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START", "METRON_ENRICHMENT_MASTER-START", "METRON_INDEXING-START", "METRON_PROFILER-START", "METRON_PCAP-START", "METRON_REST-START", "METRON_MANAGEMENT_UI-START", "METRON_ALERTS_UI-START"],
"METRON_PARSERS_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START", "NAMENODE_SERVICE_CHECK-SERVICE_CHECK", "ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK", "STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK"],
"METRON_ENRICHMENT_MASTER_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START", "NAMENODE_SERVICE_CHECK-SERVICE_CHECK", "ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK", "STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK", "HBASE_MASTER_SERVICE_CHECK-SERVICE_CHECK", "HBASE_REGIONSERVER_SERVICE_CHECK-SERVICE_CHECK"],
"METRON_INDEXING_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_INDEXING-START", "NAMENODE_SERVICE_CHECK-SERVICE_CHECK", "ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK", "STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK"],
"METRON_PROFILER_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PROFILER-START", "NAMENODE_SERVICE_CHECK-SERVICE_CHECK", "ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK", "STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK", "HBASE_MASTER_SERVICE_CHECK-SERVICE_CHECK", "HBASE_REGIONSERVER_SERVICE_CHECK-SERVICE_CHECK"],
+ "METRON_PCAP_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PCAP-START", "NAMENODE_SERVICE_CHECK-SERVICE_CHECK", "ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK", "STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK"],
"METRON_REST_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_REST-START", "ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK", "STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK"],
"METRON_MANAGEMENT_UI_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_MANAGEMENT_UI-START", "METRON_REST_SERVICE_CHECK-SERVICE_CHECK"],
"METRON_ALERTS_UI_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ALERTS_UI-START", "METRON_REST_SERVICE_CHECK-SERVICE_CHECK"]
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 644ba97..5965506 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -159,6 +159,45 @@
</component>
<component>
+ <name>METRON_PCAP</name>
+ <displayName>Metron PCAP</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/pcap_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ <configuration-dependencies>
+ <config-type>metron-rest-env</config-type>
+ </configuration-dependencies>
+ </component>
+
+ <component>
<name>METRON_INDEXING</name>
<displayName>Metron Indexing</displayName>
<category>MASTER</category>
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 0525c7f..dbad44d 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -388,6 +388,8 @@ bolt_hdfs_rotation_policy_units = config['configurations']['metron-indexing-env'
bolt_hdfs_rotation_policy_count = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count']
# Pcap
+metron_pcap_topology = 'pcap'
+pcap_input_topic = 'pcap'
pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path']
pcap_base_interim_result_path = config['configurations']['metron-rest-env']['pcap_base_interim_result_path']
pcap_final_output_path = config['configurations']['metron-rest-env']['pcap_final_output_path']
@@ -396,6 +398,8 @@ pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue']
pcap_finalizer_threadpool_size= config['configurations']['metron-rest-env']['pcap_finalizer_threadpool_size']
pcap_configured_flag_file = status_params.pcap_configured_flag_file
pcap_perm_configured_flag_file = status_params.pcap_perm_configured_flag_file
+pcap_acl_configured_flag_file = status_params.pcap_acl_configured_flag_file
+
# MapReduce
metron_user_hdfs_dir = '/user/' + metron_user
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 99f5ec0..2c711cf 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -121,6 +121,7 @@ metron_keytab_path = config['configurations']['metron-env']['metron_service_keyt
# Pcap
pcap_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_configured'
pcap_perm_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_perm_configured'
+pcap_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_acl_configured'
# MapReduce
metron_user_hdfs_dir_configured_flag_file = metron_zookeeper_config_path + '/../metron_user_hdfs_dir_configured'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_commands.py
new file mode 100644
index 0000000..2190916
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_commands.py
@@ -0,0 +1,201 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import subprocess
+import time
+
+from datetime import datetime
+from resource_management.core.exceptions import Fail
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+import metron_security
+
+
+# Wrap major operations and functionality in this class
+class PcapCommands:
+ __params = None
+ __configured = False
+ __acl_configured = False
+ __pcap_topology = None
+ __pcap_perm_configured = False
+ __pcap_topic = None
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__configured = os.path.isfile(self.__params.pcap_configured_flag_file)
+ self.__acl_configured = os.path.isfile(self.__params.pcap_acl_configured_flag_file)
+ self.__pcap_perm_configured = os.path.isfile(self.__params.pcap_perm_configured_flag_file)
+ self.__pcap_topology = params.metron_pcap_topology
+ self.__pcap_topic = params.pcap_input_topic
+
+ def __get_topics(self):
+ return [self.__pcap_topic]
+
+ def __get_kafka_acl_groups(self):
+ return ['pcap']
+
+ def is_configured(self):
+ return self.__configured
+
+ def is_acl_configured(self):
+ return self.__acl_configured
+
+ def is_hdfs_perm_configured(self):
+ return self.__pcap_perm_configured
+
+ def set_configured(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.pcap_configured_flag_file, "Setting PCAP configured to True")
+
+ def set_acl_configured(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.pcap_acl_configured_flag_file, "Setting PCAP ACL configured to true")
+
+ def set_hdfs_perm_configured(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.pcap_perm_configured_flag_file, "Setting PCAP HDFS perm configured to True")
+
+ def init_pcap(self):
+ self.init_kafka_topics()
+ self.init_hdfs_dir()
+ Logger.info("Done initializing PCAP configuration")
+
+ def init_hdfs_dir(self):
+ Logger.info("Creating HDFS locations for PCAP")
+ # Non Kerberized Metron runs under 'storm', requiring write under the 'hadoop' group.
+ # Kerberized Metron runs under it's own user.
+ ownership = 0755 if self.__params.security_enabled else 0775
+ self.__params.HdfsResource(self.__params.pcap_base_path,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ group=self.__params.hadoop_group,
+ mode=ownership,
+ )
+ self.__params.HdfsResource(self.__params.pcap_base_interim_result_path,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ group=self.__params.hadoop_group,
+ mode=ownership,
+ )
+ self.__params.HdfsResource(self.__params.pcap_final_output_path,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ group=self.__params.hadoop_group,
+ mode=ownership,
+ )
+
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topic for PCAP')
+ metron_service.init_kafka_topics(self.__params, self.__get_topics())
+
+ def init_kafka_acls(self):
+ Logger.info('Creating Kafka ACLs for PCAP')
+ metron_service.init_kafka_acls(self.__params, self.__get_topics())
+ metron_service.init_kafka_acl_groups(self.__params, self.__get_kafka_acl_groups())
+
+ def is_topology_active(self, env):
+ env.set_params(self.__params)
+ active = True
+ topologies = metron_service.get_running_topologies(self.__params)
+ is_running = False
+ if self.__pcap_topology in topologies:
+ is_running = topologies[self.__pcap_topology] in ['ACTIVE', 'REBALANCING']
+ active &= is_running
+ return active
+
+ def start_pcap_topology(self, env):
+ Logger.info('Starting Metron PCAP topology')
+ start_cmd_template = """{0}/bin/start_pcap_topology.sh"""
+ if not self.is_topology_active(env):
+ if self.__params.security_enabled:
+ metron_security.kinit(self.__params.kinit_path_local,
+ self.__params.metron_keytab_path,
+ self.__params.metron_principal_name,
+ execute_user=self.__params.metron_user)
+ start_cmd = start_cmd_template.format(self.__params.metron_home)
+ Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
+ else :
+ Logger.info('PCAP topology already started')
+
+ Logger.info('Finished starting pcap topologies')
+
+ def stop_pcap_topology(self, env):
+ Logger.info('Stopping Metron PCAP topology')
+ if self.is_topology_active(env):
+ if self.__params.security_enabled:
+ metron_security.kinit(self.__params.kinit_path_local,
+ self.__params.metron_keytab_path,
+ self.__params.metron_principal_name,
+ execute_user=self.__params.metron_user)
+ stop_cmd = 'storm kill ' + self.__pcap_topology
+ Execute(stop_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
+
+ else :
+ Logger.info('PCAP topology already stopped')
+
+ Logger.info('Finished starting PCAP topologies')
+
+ def restart_pcap_topology(self, env):
+ Logger.info('Restarting the PCAP topology')
+ self.stop_pcap_topology(env)
+
+ # Wait for old topology to be cleaned up by Storm, before starting again.
+ retries = 0
+ topology_active = self.is_topology_active(env)
+ while self.is_topology_active(env) and retries < 3:
+ Logger.info('Existing topology still active. Will wait and retry')
+ time.sleep(10)
+ retries += 1
+
+ if not topology_active:
+ Logger.info('Waiting for storm kill to complete')
+ time.sleep(30)
+ self.start_pcap_topology(env)
+ Logger.info('Done restarting the PCAP topologies')
+ else:
+ Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.')
+
+ def service_check(self, env):
+ """
+ Performs a service check for the PCAP.
+ :param env: Environment
+ """
+ Logger.info('Checking Kafka topic for PCAP')
+ metron_service.check_kafka_topics(self.__params, self.__get_topics())
+
+ Logger.info("Checking for PCAP sequence files directory in HDFS for PCAP")
+ metron_service.check_hdfs_dir_exists(self.__params, self.__params.hdfs_pcap_sequencefiles_dir)
+
+ if self.__params.security_enabled:
+ Logger.info('Checking Kafka ACLs for PCAP')
+ metron_service.check_kafka_acls(self.__params, self.__get_topics())
+ metron_service.check_kafka_acl_groups(self.__params, self.__get_kafka_acl_groups())
+
+ Logger.info("Checking for PCAP topologies")
+ if not self.is_topology_active(env):
+ raise Fail("PCAP topologies not running")
+
+ Logger.info("PCAP service check completed successfully")
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
new file mode 100644
index 0000000..a3bc1b4
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
@@ -0,0 +1,105 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import File
+from resource_management.core.source import Template
+from resource_management.libraries.functions.format import format
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.script import Script
+
+from metron_security import storm_security_setup
+import metron_service
+import metron_security
+from pcap_commands import PcapCommands
+
+
+class Pcap(Script):
+ __configured = False
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ env.set_params(params)
+
+ if not metron_service.is_zk_configured(params):
+ metron_service.init_zk_config(params)
+ metron_service.set_zk_configured(params)
+ metron_service.refresh_configs(params)
+
+ commands = PcapCommands(params)
+
+ if not commands.is_configured():
+ commands.init_kafka_topics()
+ commands.init_hdfs_dir()
+ if params.security_enabled and not commands.is_acl_configured():
+ commands.init_kafka_acls()
+ commands.set_acl_configured()
+ Logger.info("Calling security setup")
+ storm_security_setup(params)
+ if not commands.is_configured():
+ commands.set_configured()
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = PcapCommands(params)
+ if params.security_enabled:
+ metron_security.kinit(params.kinit_path_local,
+ params.metron_keytab_path,
+ params.metron_principal_name,
+ execute_user=params.metron_user)
+
+ if params.security_enabled and not commands.is_hdfs_perm_configured():
+ commands.init_hdfs_dir()
+ commands.set_hdfs_perm_configured()
+ if params.security_enabled and not commands.is_acl_configured():
+ commands.init_kafka_acls()
+ commands.set_acl_configured()
+
+ commands.start_pcap_topology(env)
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = PcapCommands(params)
+ commands.stop_pcap_topology(env)
+
+ def status(self, env):
+ from params import status_params
+ env.set_params(status_params)
+ commands = PcapCommands(status_params)
+ if not commands.is_topology_active(env):
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = PcapCommands(params)
+ commands.restart_pcap_topology(env)
+
+if __name__ == "__main__":
+ Pcap().execute()
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
index d44f478..c410b94 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@ -37,8 +37,6 @@ class RestCommands:
__kafka_acl_configured = False
__hbase_configured = False
__hbase_acl_configured = False
- __pcap_configured = False
- __pcap_perm_configured = False
__metron_user_hdfs_dir_configured = False
def __init__(self, params):
@@ -49,8 +47,6 @@ class RestCommands:
self.__kafka_acl_configured = os.path.isfile(self.__params.rest_kafka_acl_configured_flag_file)
self.__hbase_configured = os.path.isfile(self.__params.rest_hbase_configured_flag_file)
self.__hbase_acl_configured = os.path.isfile(self.__params.rest_hbase_acl_configured_flag_file)
- self.__pcap_configured = os.path.isfile(self.__params.pcap_configured_flag_file)
- self.__pcap_perm_configured = os.path.isfile(self.__params.pcap_perm_configured_flag_file)
self.__metron_user_hdfs_dir_configured = os.path.isfile(self.__params.metron_user_hdfs_dir_configured_flag_file)
Directory(params.metron_rest_pid_dir,
mode=0755,
@@ -80,12 +76,6 @@ class RestCommands:
def is_hbase_acl_configured(self):
return self.__hbase_acl_configured
- def is_pcap_configured(self):
- return self.__pcap_configured
-
- def is_pcap_perm_configured(self):
- return self.__pcap_perm_configured
-
def is_metron_user_hdfs_dir_configured(self):
return self.__metron_user_hdfs_dir_configured
@@ -101,12 +91,6 @@ class RestCommands:
def set_hbase_acl_configured(self):
metron_service.set_configured(self.__params.metron_user, self.__params.rest_hbase_acl_configured_flag_file, "Setting HBase ACL configured to True for rest")
- def set_pcap_configured(self):
- metron_service.set_configured(self.__params.metron_user, self.__params.pcap_configured_flag_file, "Setting Pcap configured to True")
-
- def set_pcap_perm_configured(self):
- metron_service.set_configured(self.__params.metron_user, self.__params.pcap_perm_configured_flag_file, "Setting Pcap perm configured to True")
-
def set_metron_user_hdfs_dir_configured(self):
metron_service.set_configured(self.__params.metron_user, self.__params.metron_user_hdfs_dir_configured_flag_file, "Setting Metron user HDFS directory configured to True")
@@ -124,33 +108,6 @@ class RestCommands:
groups = ['metron-rest']
metron_service.init_kafka_acl_groups(self.__params, groups)
- def init_pcap(self):
- Logger.info("Creating HDFS locations for Pcap")
- # Non Kerberized Metron runs under 'storm', requiring write under the 'hadoop' group.
- # Kerberized Metron runs under it's own user.
- ownership = 0755 if self.__params.security_enabled else 0775
- self.__params.HdfsResource(self.__params.pcap_base_path,
- type="directory",
- action="create_on_execute",
- owner=self.__params.metron_user,
- group=self.__params.hadoop_group,
- mode=ownership,
- )
- self.__params.HdfsResource(self.__params.pcap_base_interim_result_path,
- type="directory",
- action="create_on_execute",
- owner=self.__params.metron_user,
- group=self.__params.hadoop_group,
- mode=ownership,
- )
- self.__params.HdfsResource(self.__params.pcap_final_output_path,
- type="directory",
- action="create_on_execute",
- owner=self.__params.metron_user,
- group=self.__params.hadoop_group,
- mode=ownership,
- )
-
def create_metron_user_hdfs_dir(self):
Logger.info("Creating HDFS location for Metron user")
self.__params.HdfsResource(self.__params.metron_user_hdfs_dir,
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
index 791ca77..43224ad 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
@@ -51,8 +51,6 @@ class RestMaster(Script):
commands.init_kafka_topics()
if not commands.is_hbase_configured():
commands.create_hbase_tables()
- if not commands.is_pcap_configured():
- commands.init_pcap()
if not commands.is_metron_user_hdfs_dir_configured():
commands.create_metron_user_hdfs_dir()
if params.security_enabled and not commands.is_hbase_acl_configured():
@@ -60,11 +58,6 @@ class RestMaster(Script):
if params.security_enabled and not commands.is_kafka_acl_configured():
commands.init_kafka_acls()
commands.set_kafka_acl_configured()
- if params.security_enabled and not commands.is_pcap_perm_configured():
- # If we Kerberize the cluster, we need to call this again, to remove write perms from hadoop group
- # If we start off Kerberized, it just does the same thing twice.
- commands.init_pcap()
- commands.set_pcap_perm_configured()
def start(self, env, upgrade_type=None):
from params import params
http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
index 3bf5c7c..b008fc0 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
@@ -44,6 +44,7 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
metronParsersHost = self.getHosts(componentsList, "METRON_PARSERS")[0]
metronEnrichmentMaster = self.getHosts(componentsList, "METRON_ENRICHMENT_MASTER")[0]
metronProfilerHost = self.getHosts(componentsList, "METRON_PROFILER")[0]
+ metronPcapHost = self.getHosts(componentsList, "METRON_PCAP")[0]
metronIndexingHost = self.getHosts(componentsList, "METRON_INDEXING")[0]
metronRESTHost = self.getHosts(componentsList, "METRON_REST")[0]
metronManagementUIHost = self.getHosts(componentsList, "METRON_MANAGEMENT_UI")[0]
@@ -83,6 +84,10 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
message = "Metron Indexing must be co-located with Metron Parsers on {0}".format(metronParsersHost)
items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_INDEXING', "host": metronIndexingHost })
+ if metronParsersHost != metronPcapHost:
+ message = "Metron PCAP must be co-located with Metron Parsers on {0}".format(metronParsersHost)
+ items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_PCAP', "host": metronPcapHost })
+
if metronParsersHost != metronProfilerHost:
message = "Metron Profiler must be co-located with Metron Parsers on {0}".format(metronParsersHost)
items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_PROFILER', "host": metronProfilerHost })
[08/51] [abbrv] metron git commit: METRON-1770 Add Docs for Running
the Profiler with Spark on YARN (nickwallen) closes apache/metron#1189
Posted by ot...@apache.org.
METRON-1770 Add Docs for Running the Profiler with Spark on YARN (nickwallen) closes apache/metron#1189
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f83f0ac0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f83f0ac0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f83f0ac0
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: f83f0ac06622e091a09d9f256f817e7235c63e53
Parents: cad2f40
Author: nickwallen <ni...@nickallen.org>
Authored: Wed Sep 19 10:01:50 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Wed Sep 19 10:01:50 2018 -0400
----------------------------------------------------------------------
.../metron-profiler-spark/README.md | 94 ++++++++++++++------
.../src/main/config/batch-profiler.properties | 8 +-
2 files changed, 76 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/f83f0ac0/metron-analytics/metron-profiler-spark/README.md
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/README.md b/metron-analytics/metron-profiler-spark/README.md
index d137e51..3d7017c 100644
--- a/metron-analytics/metron-profiler-spark/README.md
+++ b/metron-analytics/metron-profiler-spark/README.md
@@ -22,8 +22,8 @@ This project allows profiles to be executed using [Apache Spark](https://spark.a
* [Introduction](#introduction)
* [Getting Started](#getting-started)
* [Installation](#installation)
-* [Configuring the Profiler](#configuring-the-profiler)
* [Running the Profiler](#running-the-profiler)
+* [Configuring the Profiler](#configuring-the-profiler)
## Introduction
@@ -129,6 +129,73 @@ The Batch Profiler requires Spark version 2.3.0+.
find ./ -name "metron-profiler-spark*.deb"
```
+## Running the Profiler
+
+A script located at `$METRON_HOME/bin/start_batch_profiler.sh` has been provided to simplify running the Batch Profiler. This script makes the following assumptions.
+
+ * The script builds the profiles defined in `$METRON_HOME/config/zookeeper/profiler.json`.
+
+ * The properties defined in `$METRON_HOME/config/batch-profiler.properties` are passed to both the Profiler and Spark. You can define both Spark and Profiler properties in this same file.
+
+ * The script assumes that Spark is installed at `/usr/hdp/current/spark2-client`. This can be overridden if you define an environment variable called `SPARK_HOME` prior to executing the script.
+
+### Advanced Usage
+
+The Batch Profiler may also be started using `spark-submit` as follows. See the Spark Documentation for more information about [`spark-submit`](https://spark.apache.org/docs/latest/submitting-applications.html#launching-applications-with-spark-submit).
+
+```
+${SPARK_HOME}/bin/spark-submit \
+ --class org.apache.metron.profiler.spark.cli.BatchProfilerCLI \
+ --properties-file ${SPARK_PROPS_FILE} \
+ ${METRON_HOME}/lib/metron-profiler-spark-*.jar \
+ --config ${PROFILER_PROPS_FILE} \
+ --profiles ${PROFILES_FILE}
+```
+
+The Batch Profiler accepts the following arguments when run from the command line as shown above. All arguments following the Profiler jar are passed to the Profiler. All argument preceeding the Profiler jar are passed to Spark.
+
+| Argument | Description
+|--- |---
+| -p, --profiles | The path to a file containing the profile definitions.
+| -c, --config | The path to the profiler properties file.
+| -g, --globals | The path to a properties file containing global properties.
+| -h, --help | Print the help text.
+
+### Spark Execution
+
+Spark supports a number of different [cluster managers](https://spark.apache.org/docs/latest/cluster-overview.html#cluster-manager-types). The underlying cluster manager is transparent to the Profiler. To run the Profiler on a particular cluster manager, it is just a matter of setting the appropriate options as defined in the Spark documentation.
+
+#### Local Mode
+
+By default, the Batch Profiler instructs Spark to run in local mode. This will run all of the Spark execution components within a single JVM. This mode is only useful for testing with a limited set of data.
+
+`$METRON_HOME/config/batch-profiler.properties`
+```
+spark.master=local
+```
+
+#### Spark on YARN
+
+To run the Profiler using [Spark on YARN](https://spark.apache.org/docs/latest/running-on-yarn.html#running-spark-on-yarn), at a minimum edit the value of `spark.master` as shown. In many cases it also makes sense to set the YARN [deploy mode](https://spark.apache.org/docs/latest/running-on-yarn.html#launching-spark-on-yarn) to `cluster`.
+
+`$METRON_HOME/config/batch-profiler.properties`
+```
+spark.master=yarn
+spark.submit.deployMode=cluster
+```
+
+See the Spark documentation for information on how to further control the execution of Spark on YARN. Any of [these properties](http://spark.apache.org/docs/latest/running-on-yarn.html#spark-properties) can be added to the Profiler properties file.
+
+The following command can be useful to review the logs generated when the Profiler is executed on YARN.
+```
+yarn logs -applicationId <application-id>
+```
+
+#### Kerberos
+
+See the Spark documentation for information on running the Batch Profiler in a [secure, kerberized cluster](https://spark.apache.org/docs/latest/running-on-yarn.html#running-in-a-secure-cluster).
+
+
## Configuring the Profiler
By default, the configuration for the Batch Profiler is stored in the local filesystem at `$METRON_HOME/config/batch-profiler.properties`.
@@ -147,7 +214,7 @@ You can store both settings for the Profiler along with settings for Spark in th
### `profiler.batch.input.path`
-*Default*: "hdfs://localhost:9000/apps/metron/indexing/indexed/*/*"
+*Default*: hdfs://localhost:9000/apps/metron/indexing/indexed/\*/\*
The path to the input data read by the Batch Profiler.
@@ -190,26 +257,3 @@ The name of the HBase table that profile data is written to. The Profiler expec
*Default*: P
The column family used to store profile data in HBase.
-
-## Running the Profiler
-
-A script located at `$METRON_HOME/bin/start_batch_profiler.sh` has been provided to simplify running the Batch Profiler. The Batch Profiler may also be started as follows using the `spark-submit` script.
-
-```
-${SPARK_HOME}/bin/spark-submit \
- --class org.apache.metron.profiler.spark.cli.BatchProfilerCLI \
- --properties-file ${SPARK_PROPS_FILE} \
- ${PROFILER_JAR} \
- --config ${PROFILER_PROPS_FILE} \
- --profiles ${PROFILES_FILE}
-```
-
-The Batch Profiler also accepts the following command line arguments when run from the command line.
-
-| Argument | Description
-|--- |---
-| -p, --profiles | The path to a file containing the profile definitions.
-| -c, --config | The path to the profiler properties file.
-| -g, --globals | The path to a properties file containing global properties.
-| -h, --help | Print the help text.
-
http://git-wip-us.apache.org/repos/asf/metron/blob/f83f0ac0/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties b/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties
index c651791..400c526 100644
--- a/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties
+++ b/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties
@@ -16,5 +16,11 @@
# limitations under the License.
#
#
-spark.master=local
spark.app.name=Batch Profiler
+spark.master=local
+
+profiler.batch.input.path=hdfs://localhost:9000/apps/metron/indexing/indexed/*/*
+profiler.batch.input.format=text
+
+profiler.period.duration=15
+profiler.period.duration.units=MINUTES
[09/51] [abbrv] metron git commit: METRON-1772 Support alternative
input formats in the Batch Profiler (nickwallen) closes apache/metron#1191
Posted by ot...@apache.org.
METRON-1772 Support alternative input formats in the Batch Profiler (nickwallen) closes apache/metron#1191
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1545978e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1545978e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1545978e
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 1545978e169a01e4a06735b8713c8fa65373a394
Parents: f83f0ac
Author: nickwallen <ni...@nickallen.org>
Authored: Wed Sep 19 10:11:28 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Wed Sep 19 10:11:28 2018 -0400
----------------------------------------------------------------------
.../metron-profiler-spark/README.md | 47 +++++++++-
metron-analytics/metron-profiler-spark/pom.xml | 18 ++--
.../metron/profiler/spark/BatchProfiler.java | 21 +++--
.../profiler/spark/cli/BatchProfilerCLI.java | 40 +++++++--
.../spark/cli/BatchProfilerCLIOptions.java | 10 ++-
.../spark/BatchProfilerIntegrationTest.java | 91 +++++++++++++++++---
6 files changed, 189 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/1545978e/metron-analytics/metron-profiler-spark/README.md
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/README.md b/metron-analytics/metron-profiler-spark/README.md
index 3d7017c..99e8c7e 100644
--- a/metron-analytics/metron-profiler-spark/README.md
+++ b/metron-analytics/metron-profiler-spark/README.md
@@ -131,6 +131,14 @@ The Batch Profiler requires Spark version 2.3.0+.
## Running the Profiler
+* [Usage](#usage)
+* [Advanced Usage](#advanced-usage)
+* [Spark Execution](#spark-execution)
+* [Kerberos](#kerberos)
+* [Input Formats](#input-formats)
+
+### Usage
+
A script located at `$METRON_HOME/bin/start_batch_profiler.sh` has been provided to simplify running the Batch Profiler. This script makes the following assumptions.
* The script builds the profiles defined in `$METRON_HOME/config/zookeeper/profiler.json`.
@@ -156,11 +164,28 @@ The Batch Profiler accepts the following arguments when run from the command lin
| Argument | Description
|--- |---
-| -p, --profiles | The path to a file containing the profile definitions.
-| -c, --config | The path to the profiler properties file.
-| -g, --globals | The path to a properties file containing global properties.
+| -p, --profiles | Path to the profile definitions.
+| -c, --config | Path to the profiler properties file.
+| -g, --globals | Path to the Stellar global config file.
+| -r, --reader | Path to properties for the DataFrameReader.
| -h, --help | Print the help text.
+#### `--profiles`
+
+The path to a file containing the profile definition in JSON.
+
+#### `--config`
+
+The path to a file containing key-value properties for the Profiler. This file would contain the properties described under [Configuring the Profiler](#configuring-the-profiler).
+
+#### `--globals`
+
+The path to a file containing key-value properties that define the global properties. This can be used to customize how certain Stellar functions behave during execution.
+
+#### `--reader`
+
+The path to a file containing key-value properties that are passed to the DataFrameReader when reading the input telemetry. This allows additional customization for how the input telemetry is read.
+
### Spark Execution
Spark supports a number of different [cluster managers](https://spark.apache.org/docs/latest/cluster-overview.html#cluster-manager-types). The underlying cluster manager is transparent to the Profiler. To run the Profiler on a particular cluster manager, it is just a matter of setting the appropriate options as defined in the Spark documentation.
@@ -191,10 +216,24 @@ The following command can be useful to review the logs generated when the Profil
yarn logs -applicationId <application-id>
```
-#### Kerberos
+### Kerberos
See the Spark documentation for information on running the Batch Profiler in a [secure, kerberized cluster](https://spark.apache.org/docs/latest/running-on-yarn.html#running-in-a-secure-cluster).
+### Input Formats
+
+The Profiler can consume archived telemetry stored in a variety of input formats. By default, it is configured to consume the text/json that Metron archives in HDFS. This is often not the best format for archiving telemetry. If you choose a different format, you should be able to configure the Profiler to consume it by doing the following.
+
+1. Edit [`profiler.batch.input.format`](#profilerbatchinputformat) and [`profiler.batch.input.path`](#profilerbatchinputpath) as needed. For example, to read ORC you might do the following.
+
+ `$METRON_HOME/config/batch-profiler.properties`
+ ```
+ profiler.batch.input.format=org.apache.spark.sql.execution.datasources.orc
+ profiler.batch.input.path=hdfs://localhost:9000/apps/metron/indexing/orc/\*/\*
+ ```
+
+1. If additional options are required for your input format, then use the [`--reader`](#--reader) command-line argument when launching the Batch Profiler as [described here](#advanced-usage).
+
## Configuring the Profiler
http://git-wip-us.apache.org/repos/asf/metron/blob/1545978e/metron-analytics/metron-profiler-spark/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/pom.xml b/metron-analytics/metron-profiler-spark/pom.xml
index 587b38c..668ee2c 100644
--- a/metron-analytics/metron-profiler-spark/pom.xml
+++ b/metron-analytics/metron-profiler-spark/pom.xml
@@ -25,6 +25,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ <spark_antlr_version>4.7</spark_antlr_version>
</properties>
<dependencies>
<dependency>
@@ -36,12 +37,11 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>${global_spark_version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.antlr</groupId>
- <artifactId>antlr-runtime</artifactId>
- </exclusion>
- </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr4-runtime</artifactId>
+ <version>${spark_antlr_version}</version>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
@@ -53,6 +53,12 @@
<artifactId>metron-profiler-client</artifactId>
<version>${project.parent.version}</version>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr4-runtime</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
http://git-wip-us.apache.org/repos/asf/metron/blob/1545978e/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
index f999613..d75abc3 100644
--- a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
@@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
@@ -54,25 +55,29 @@ public class BatchProfiler implements Serializable {
* Execute the Batch Profiler.
*
* @param spark The spark session.
- * @param properties The profiler configuration properties.
+ * @param profilerProps The profiler configuration properties.
+ * @param globalProperties The Stellar global properties.
+ * @param readerProps The properties passed to the {@link org.apache.spark.sql.DataFrameReader}.
* @param profiles The profile definitions.
* @return The number of profile measurements produced.
*/
public long run(SparkSession spark,
- Properties properties,
+ Properties profilerProps,
Properties globalProperties,
+ Properties readerProps,
ProfilerConfig profiles) {
LOG.debug("Building {} profile(s)", profiles.getProfiles().size());
Map<String, String> globals = Maps.fromProperties(globalProperties);
- String inputFormat = TELEMETRY_INPUT_FORMAT.get(properties, String.class);
- String inputPath = TELEMETRY_INPUT_PATH.get(properties, String.class);
+ String inputFormat = TELEMETRY_INPUT_FORMAT.get(profilerProps, String.class);
+ String inputPath = TELEMETRY_INPUT_PATH.get(profilerProps, String.class);
LOG.debug("Loading telemetry from '{}'", inputPath);
// fetch the archived telemetry
Dataset<String> telemetry = spark
.read()
+ .options(Maps.fromProperties(readerProps))
.format(inputFormat)
.load(inputPath)
.as(Encoders.STRING());
@@ -85,13 +90,13 @@ public class BatchProfiler implements Serializable {
// build the profiles
Dataset<ProfileMeasurementAdapter> measurements = routes
- .groupByKey(new GroupByPeriodFunction(properties), Encoders.STRING())
- .mapGroups(new ProfileBuilderFunction(properties, globals), Encoders.bean(ProfileMeasurementAdapter.class));
+ .groupByKey(new GroupByPeriodFunction(profilerProps), Encoders.STRING())
+ .mapGroups(new ProfileBuilderFunction(profilerProps, globals), Encoders.bean(ProfileMeasurementAdapter.class));
LOG.debug("Produced {} profile measurement(s)", measurements.cache().count());
// write the profile measurements to HBase
long count = measurements
- .mapPartitions(new HBaseWriterFunction(properties), Encoders.INT())
+ .mapPartitions(new HBaseWriterFunction(profilerProps), Encoders.INT())
.agg(sum("value"))
.head()
.getLong(0);
@@ -99,4 +104,4 @@ public class BatchProfiler implements Serializable {
return count;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/1545978e/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java
index bdcf231..29fe4a2 100644
--- a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java
@@ -37,9 +37,10 @@ import java.io.Serializable;
import java.lang.invoke.MethodHandles;
import java.util.Properties;
-import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.CONFIGURATION_FILE;
+import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.PROFILER_PROPS_FILE;
import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.GLOBALS_FILE;
import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.PROFILE_DEFN_FILE;
+import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.READER_PROPS_FILE;
import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.parse;
/**
@@ -54,7 +55,8 @@ import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.parse
* metron-profiler-spark-<version>.jar \
* --config profiler.properties \
* --globals global.properties \
- * --profiles profiles.json
+ * --profiles profiles.json \
+ * --reader reader.properties
* }</pre>
*/
public class BatchProfilerCLI implements Serializable {
@@ -63,6 +65,7 @@ public class BatchProfilerCLI implements Serializable {
public static Properties globals;
public static Properties profilerProps;
+ public static Properties readerProps;
public static ProfilerConfig profiles;
public static void main(String[] args) throws IOException, org.apache.commons.cli.ParseException {
@@ -71,6 +74,7 @@ public class BatchProfilerCLI implements Serializable {
profilerProps = handleProfilerProperties(commandLine);
globals = handleGlobals(commandLine);
profiles = handleProfileDefinitions(commandLine);
+ readerProps = handleReaderProperties(commandLine);
// the batch profiler must use 'event time'
if(!profiles.getTimestampField().isPresent()) {
@@ -88,7 +92,7 @@ public class BatchProfilerCLI implements Serializable {
.getOrCreate();
BatchProfiler profiler = new BatchProfiler();
- long count = profiler.run(spark, profilerProps, globals, profiles);
+ long count = profiler.run(spark, profilerProps, globals, readerProps, profiles);
LOG.info("Profiler produced {} profile measurement(s)", count);
}
@@ -117,13 +121,31 @@ public class BatchProfilerCLI implements Serializable {
*/
private static Properties handleProfilerProperties(CommandLine commandLine) throws IOException {
Properties config = new Properties();
- if(CONFIGURATION_FILE.has(commandLine)) {
- String propertiesPath = CONFIGURATION_FILE.get(commandLine);
+ if(PROFILER_PROPS_FILE.has(commandLine)) {
+ String propertiesPath = PROFILER_PROPS_FILE.get(commandLine);
LOG.info("Loading profiler properties from '{}'", propertiesPath);
config.load(new FileInputStream(propertiesPath));
- LOG.info("Properties = {}", config.toString());
+ LOG.info("Profiler properties = {}", config.toString());
+ }
+ return config;
+ }
+
+ /**
+ * Load the properties for the {@link org.apache.spark.sql.DataFrameReader}.
+ *
+ * @param commandLine The command line.
+ */
+ private static Properties handleReaderProperties(CommandLine commandLine) throws IOException {
+ Properties config = new Properties();
+ if(READER_PROPS_FILE.has(commandLine)) {
+ String readerPropsPath = READER_PROPS_FILE.get(commandLine);
+
+ LOG.info("Loading reader properties from '{}'", readerPropsPath);
+ config.load(new FileInputStream(readerPropsPath));
+
+ LOG.info("Reader properties = {}", config.toString());
}
return config;
}
@@ -171,4 +193,8 @@ public class BatchProfilerCLI implements Serializable {
public static ProfilerConfig getProfiles() {
return profiles;
}
-}
\ No newline at end of file
+
+ public static Properties getReaderProps() {
+ return readerProps;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/1545978e/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java
index f5dfe12..d58728a 100644
--- a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java
@@ -36,12 +36,12 @@ import java.util.function.Supplier;
public enum BatchProfilerCLIOptions {
PROFILE_DEFN_FILE(() -> {
- Option o = new Option("p", "profiles", true, "Path to a file containing profile definitions.");
+ Option o = new Option("p", "profiles", true, "Path to the profile definitions.");
o.setRequired(true);
return o;
}),
- CONFIGURATION_FILE(() -> {
+ PROFILER_PROPS_FILE(() -> {
Option o = new Option("c", "config", true, "Path to the profiler properties file.");
o.setRequired(false);
return o;
@@ -53,6 +53,12 @@ public enum BatchProfilerCLIOptions {
return o;
}),
+ READER_PROPS_FILE(() -> {
+ Option o = new Option("r", "reader", true, "Path to properties for the DataFrameReader.");
+ o.setRequired(false);
+ return o;
+ }),
+
HELP(() -> {
Option o = new Option("h", "help", false, "Usage instructions.");
o.setRequired(false);
http://git-wip-us.apache.org/repos/asf/metron/blob/1545978e/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
index 376623c..87c4246 100644
--- a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
@@ -30,11 +30,14 @@ import org.apache.metron.stellar.common.StellarStatefulExecutor;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver;
import org.apache.spark.SparkConf;
+import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.SparkSession;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import java.io.IOException;
import java.util.Collections;
@@ -82,8 +85,12 @@ public class BatchProfilerIntegrationTest {
private static String profileJson;
private static SparkSession spark;
private Properties profilerProperties;
+ private Properties readerProperties;
private StellarStatefulExecutor executor;
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
@BeforeClass
public static void setupSpark() {
SparkConf conf = new SparkConf()
@@ -105,12 +112,9 @@ public class BatchProfilerIntegrationTest {
@Before
public void setup() {
+ readerProperties = new Properties();
profilerProperties = new Properties();
- // the input telemetry is read from the local filesystem
- profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), "src/test/resources/telemetry.json");
- profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "text");
-
// the output will be written to a mock HBase table
String tableName = HBASE_TABLE_NAME.get(profilerProperties, String.class);
String columnFamily = HBASE_COLUMN_FAMILY.get(profilerProperties, String.class);
@@ -147,15 +151,80 @@ public class BatchProfilerIntegrationTest {
* produced will center around this date.
*/
@Test
- public void testBatchProfiler() throws Exception {
- // run the batch profiler
+ public void testBatchProfilerWithJSON() throws Exception {
+ // the input telemetry is text/json stored in the local filesystem
+ profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), "src/test/resources/telemetry.json");
+ profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "text");
+
+ BatchProfiler profiler = new BatchProfiler();
+ profiler.run(spark, profilerProperties, getGlobals(), readerProperties, getProfile());
+
+ validateProfiles();
+ }
+
+ @Test
+ public void testBatchProfilerWithORC() throws Exception {
+ // re-write the test data as ORC
+ String pathToORC = tempFolder.getRoot().getAbsolutePath();
+ spark.read()
+ .format("text")
+ .load("src/test/resources/telemetry.json")
+ .as(Encoders.STRING())
+ .write()
+ .mode("overwrite")
+ .format("org.apache.spark.sql.execution.datasources.orc")
+ .save(pathToORC);
+
+ // tell the profiler to use the ORC input data
+ profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToORC);
+ profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "org.apache.spark.sql.execution.datasources.orc");
+
+ BatchProfiler profiler = new BatchProfiler();
+ profiler.run(spark, profilerProperties, getGlobals(), readerProperties, getProfile());
+
+ validateProfiles();
+ }
+
+ @Test
+ public void testBatchProfilerWithCSV() throws Exception {
+ // re-write the test data as a CSV with a header record
+ String pathToCSV = tempFolder.getRoot().getAbsolutePath();
+ spark.read()
+ .format("text")
+ .load("src/test/resources/telemetry.json")
+ .as(Encoders.STRING())
+ .write()
+ .mode("overwrite")
+ .option("header", "true")
+ .format("csv")
+ .save(pathToCSV);
+
+ // tell the profiler to use the CSV input data
+ profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToCSV);
+ profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "csv");
+
+ // set a reader property; tell the reader to expect a header
+ readerProperties.put("header", "true");
+
BatchProfiler profiler = new BatchProfiler();
- profiler.run(spark, profilerProperties, getGlobals(), getProfile());
+ profiler.run(spark, profilerProperties, getGlobals(), readerProperties, getProfile());
+
+ validateProfiles();
+ }
+
+ /**
+ * Validates the profiles that were built.
+ *
+ * These tests use the Batch Profiler to seed two profiles with archived telemetry. The first profile
+ * called 'count-by-ip', counts the number of messages by 'ip_src_addr'. The second profile called
+ * 'total-count', counts the total number of messages.
+ */
+ private void validateProfiles() {
+ // the max timestamp in the data is around July 7, 2018
+ assign("maxTimestamp", "1530978728982L");
- // validate the measurements written by the batch profiler using `PROFILE_GET`
- // the 'window' looks up to 5 hours before the last timestamp contained in the telemetry
- assign("lastTimestamp", "1530978728982L");
- assign("window", "PROFILE_WINDOW('from 5 hours ago', lastTimestamp)");
+ // the 'window' looks up to 5 hours before the max timestamp
+ assign("window", "PROFILE_WINDOW('from 5 hours ago', maxTimestamp)");
// there are 26 messages where ip_src_addr = 192.168.66.1
assertTrue(execute("[26] == PROFILE_GET('count-by-ip', '192.168.66.1', window)", Boolean.class));
[42/51] [abbrv] metron git commit: METRON-1820 Update to new
Simple-Syslog-5424 version to support error handling (ottobackwards) closes
apache/metron#1234
Posted by ot...@apache.org.
METRON-1820 Update to new Simple-Syslog-5424 version to support error handling (ottobackwards) closes apache/metron#1234
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/08f3de0f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/08f3de0f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/08f3de0f
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 08f3de0fe31fefa828952cbe76456580a4697630
Parents: 6214150
Author: ottobackwards <ot...@gmail.com>
Authored: Mon Oct 15 16:36:36 2018 -0400
Committer: otto <ot...@apache.org>
Committed: Mon Oct 15 16:36:36 2018 -0400
----------------------------------------------------------------------
dependencies_with_url.csv | 2 +-
.../impl/SensorParserConfigServiceImpl.java | 65 +++++++++++++-
.../org/apache/metron/parsers/GrokParser.java | 18 +---
.../apache/metron/parsers/bolt/ParserBolt.java | 37 +++++++-
.../parsers/interfaces/MessageParser.java | 18 ----
.../interfaces/MultilineMessageParser.java | 51 +++++++++++
.../metron/parsers/syslog/Syslog5424Parser.java | 44 ++++++++--
.../apache/metron/parsers/GrokParserTest.java | 8 +-
.../metron/parsers/MultiLineGrokParserTest.java | 5 +-
.../metron/parsers/bolt/ParserBoltTest.java | 91 +++++++++++---------
.../parsers/syslog/Syslog5424ParserTest.java | 55 +++++++++---
.../websphere/GrokWebSphereParserTest.java | 56 ++++++++----
pom.xml | 2 +-
13 files changed, 335 insertions(+), 117 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index fb6c03c..2e1eedd 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -484,4 +484,4 @@ org.sonatype.sisu:sisu-inject-bean:jar:2.2.2:compile
org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
com.zaxxer:HikariCP:jar:2.7.8:compile,ASLv2,https://github.com/brettwooldridge/HikariCP
org.hibernate.validator:hibernate-validator:jar:6.0.9.Final:compile,ASLv2,https://github.com/hibernate/hibernate-validator
-com.github.palindromicity:simple-syslog-5424:jar:0.0.8:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
+com.github.palindromicity:simple-syslog-5424:jar:0.0.9:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
index 85b84b8..4cd272e 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.fs.Path;
@@ -33,6 +34,8 @@ import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.apache.metron.parsers.interfaces.MultilineMessageParser;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.ParseMessageRequest;
@@ -138,13 +141,53 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
} else if (sensorParserConfig.getParserClassName() == null) {
throw new RestException("SensorParserConfig must have a parserClassName");
} else {
- MessageParser<JSONObject> parser;
+ MultilineMessageParser<JSONObject> parser;
+ Object parserObject;
try {
- parser = (MessageParser<JSONObject>) Class.forName(sensorParserConfig.getParserClassName())
+ parserObject = Class.forName(sensorParserConfig.getParserClassName())
.newInstance();
} catch (Exception e) {
throw new RestException(e.toString(), e.getCause());
}
+
+ if (!(parserObject instanceof MultilineMessageParser)) {
+ parser = new MultilineMessageParser<JSONObject>() {
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void configure(Map<String, Object> config) {
+ ((MessageParser<JSONObject>)parserObject).configure(config);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void init() {
+ ((MessageParser<JSONObject>)parserObject).init();
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public boolean validate(JSONObject message) {
+ return ((MessageParser<JSONObject>)parserObject).validate(message);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public List<JSONObject> parse(byte[] message) {
+ return ((MessageParser<JSONObject>)parserObject).parse(message);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Optional<List<JSONObject>> parseOptional(byte[] message) {
+ return ((MessageParser<JSONObject>)parserObject).parseOptional(message);
+ }
+ };
+ } else {
+ parser = (MultilineMessageParser<JSONObject>)parserObject;
+ }
+
+
Path temporaryGrokPath = null;
if (isGrokConfig(sensorParserConfig)) {
String name = parseMessageRequest.getSensorParserConfig().getSensorTopic();
@@ -152,13 +195,27 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
sensorParserConfig.getParserConfig()
.put(MetronRestConstants.GROK_PATH_KEY, new Path(temporaryGrokPath, name).toString());
}
+
parser.configure(sensorParserConfig.getParserConfig());
parser.init();
- JSONObject results = parser.parse(parseMessageRequest.getSampleData().getBytes()).get(0);
+
+ Optional<MessageParserResult<JSONObject>> result = parser.parseOptionalResult(parseMessageRequest.getSampleData().getBytes());
+ if (!result.isPresent()) {
+ throw new RestException("Unknown error parsing sample data");
+ }
+
+ if (result.get().getMasterThrowable().isPresent()) {
+ throw new RestException("Error parsing sample data",result.get().getMasterThrowable().get());
+ }
+
+ if (result.get().getMessages().isEmpty()) {
+ throw new RestException("No results from parsing sample data");
+ }
+
if (isGrokConfig(sensorParserConfig) && temporaryGrokPath != null) {
grokService.deleteTemporary();
}
- return results;
+ return result.get().getMessages().get(0);
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
index bead477..a81149d 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.metron.common.Constants;
import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.apache.metron.parsers.interfaces.MultilineMessageParser;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,7 +54,7 @@ import java.util.Optional;
import java.util.TimeZone;
-public class GrokParser implements MessageParser<JSONObject>, Serializable {
+public class GrokParser implements MultilineMessageParser<JSONObject>, Serializable {
protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -143,21 +144,6 @@ public class GrokParser implements MessageParser<JSONObject>, Serializable {
@SuppressWarnings("unchecked")
@Override
- public List<JSONObject> parse(byte[] rawMessage) {
- Optional<MessageParserResult<JSONObject>> resultOptional = parseOptionalResult(rawMessage);
- if (!resultOptional.isPresent()) {
- return Collections.EMPTY_LIST;
- }
- Map<Object,Throwable> errors = resultOptional.get().getMessageThrowables();
- if (!errors.isEmpty()) {
- throw new RuntimeException(errors.entrySet().iterator().next().getValue());
- }
-
- return resultOptional.get().getMessages();
- }
-
- @SuppressWarnings("unchecked")
- @Override
public Optional<MessageParserResult<JSONObject>> parseOptionalResult(byte[] rawMessage) {
if (grok == null) {
init();
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
index ff5c1d4..05334c2 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
@@ -37,6 +37,7 @@ import org.apache.metron.parsers.filters.Filters;
import org.apache.metron.parsers.interfaces.MessageFilter;
import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.apache.metron.parsers.interfaces.MultilineMessageParser;
import org.apache.metron.parsers.topology.ParserComponents;
import org.apache.metron.stellar.common.CachingStellarProcessor;
import org.apache.metron.stellar.dsl.Context;
@@ -307,9 +308,43 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable {
, sensorParserConfig.getReadMetadata()
, sensorParserConfig.getRawMessageStrategyConfig()
);
+
metadata = rawMessage.getMetadata();
- Optional<MessageParserResult<JSONObject>> results = parser.parseOptionalResult(rawMessage.getMessage());
+ MultilineMessageParser mmp = null;
+ if (!(parser instanceof MultilineMessageParser)) {
+ mmp = new MultilineMessageParser<JSONObject>() {
+
+ @Override
+ public void configure(Map<String, Object> config) {
+ parser.configure(config);
+ }
+
+ @Override
+ public void init() {
+ parser.init();
+ }
+
+ @Override
+ public boolean validate(JSONObject message) {
+ return parser.validate(message);
+ }
+
+ @Override
+ public List<JSONObject> parse(byte[] message) {
+ return parser.parse(message);
+ }
+
+ @Override
+ public Optional<List<JSONObject>> parseOptional(byte[] message) {
+ return parser.parseOptional(message);
+ }
+ };
+ } else {
+ mmp = (MultilineMessageParser) parser;
+ }
+
+ Optional<MessageParserResult<JSONObject>> results = mmp.parseOptionalResult(rawMessage.getMessage());
// check if there is a master error
if (results.isPresent() && results.get().getMasterThrowable().isPresent()) {
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
index f8243b9..665076b 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
@@ -50,24 +50,6 @@ public interface MessageParser<T> extends Configurable {
}
/**
- * Take raw data and convert it to messages. Each raw message may produce multiple messages and therefore
- * multiple errors. A {@link MessageParserResult} is returned, which will have both the messages produced
- * and the errors.
- * @param parseMessage the raw bytes of the message
- * @return Optional of {@link MessageParserResult}
- */
- default Optional<MessageParserResult<T>> parseOptionalResult(byte[] parseMessage) {
- List<T> list = new ArrayList<>();
- try {
- Optional<List<T>> optionalMessages = parseOptional(parseMessage);
- optionalMessages.ifPresent(list::addAll);
- } catch (Throwable t) {
- return Optional.of(new DefaultMessageParserResult<>(t));
- }
- return Optional.of(new DefaultMessageParserResult<T>(list));
- }
-
- /**
* Validate the message to ensure that it's correct.
* @param message the message to validate
* @return true if the message is valid, false if not
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MultilineMessageParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MultilineMessageParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MultilineMessageParser.java
new file mode 100644
index 0000000..7818f9a
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MultilineMessageParser.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.interfaces;
+
+import org.apache.commons.lang3.NotImplementedException;
+import org.apache.metron.parsers.DefaultMessageParserResult;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+public interface MultilineMessageParser<T> extends MessageParser<T> {
+
+ default List<T> parse(byte[] rawMessage) {
+ throw new NotImplementedException("parse is not implemented");
+ }
+
+ /**
+ * Take raw data and convert it to messages. Each raw message may produce multiple messages and therefore
+ * multiple errors. A {@link MessageParserResult} is returned, which will have both the messages produced
+ * and the errors.
+ * @param parseMessage the raw bytes of the message
+ * @return Optional of {@link MessageParserResult}
+ */
+ default Optional<MessageParserResult<T>> parseOptionalResult(byte[] parseMessage) {
+ List<T> list = new ArrayList<>();
+ try {
+ Optional<List<T>> optionalMessages = parseOptional(parseMessage);
+ optionalMessages.ifPresent(list::addAll);
+ } catch (Throwable t) {
+ return Optional.of(new DefaultMessageParserResult<>(t));
+ }
+ return Optional.of(new DefaultMessageParserResult<T>(list));
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
index e3ad941..79a082a 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
@@ -25,23 +25,35 @@ import com.github.palindromicity.syslog.SyslogParserBuilder;
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys;
import org.apache.commons.lang3.StringUtils;
import org.apache.metron.parsers.BasicParser;
+import org.apache.metron.parsers.DefaultMessageParserResult;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.apache.metron.parsers.interfaces.MultilineMessageParser;
import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
+import java.io.IOException;
import java.io.Reader;
+import java.io.Serializable;
import java.io.StringReader;
+import java.lang.invoke.MethodHandles;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
/**
* Parser for well structured RFC 5424 messages.
*/
-public class Syslog5424Parser extends BasicParser {
+public class Syslog5424Parser implements MultilineMessageParser<JSONObject>, Serializable {
+ protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String NIL_POLICY_CONFIG = "nilPolicy";
private transient SyslogParser syslogParser;
@@ -62,15 +74,31 @@ public class Syslog5424Parser extends BasicParser {
}
@Override
+ public boolean validate(JSONObject message) {
+ JSONObject value = message;
+ if (!(value.containsKey("original_string"))) {
+ LOG.trace("[Metron] Message does not have original_string: {}", message);
+ return false;
+ } else if (!(value.containsKey("timestamp"))) {
+ LOG.trace("[Metron] Message does not have timestamp: {}", message);
+ return false;
+ } else {
+ LOG.trace("[Metron] Message conforms to schema: {}", message);
+ return true;
+ }
+ }
+
+ @Override
@SuppressWarnings("unchecked")
- public List<JSONObject> parse(byte[] rawMessage) {
+ public Optional<MessageParserResult<JSONObject>> parseOptionalResult(byte[] rawMessage) {
try {
if (rawMessage == null || rawMessage.length == 0) {
- return null;
+ return Optional.empty();
}
String originalString = new String(rawMessage);
List<JSONObject> returnList = new ArrayList<>();
+ Map<Object,Throwable> errorMap = new HashMap<>();
try (Reader reader = new BufferedReader(new StringReader(originalString))) {
syslogParser.parseLines(reader, (m) -> {
JSONObject jsonObject = new JSONObject(m);
@@ -79,14 +107,14 @@ public class Syslog5424Parser extends BasicParser {
jsonObject.put("original_string", originalString);
setTimestamp(jsonObject);
returnList.add(jsonObject);
- });
+ },errorMap::put);
- return returnList;
+ return Optional.of(new DefaultMessageParserResult<JSONObject>(returnList,errorMap));
}
- } catch (Exception e) {
- String message = "Unable to parse " + new String(rawMessage) + ": " + e.getMessage();
+ } catch (IOException e) {
+ String message = "Unable to read buffer " + new String(rawMessage) + ": " + e.getMessage();
LOG.error(message, e);
- throw new IllegalStateException(message, e);
+ return Optional.of(new DefaultMessageParserResult<JSONObject>( new IllegalStateException(message, e)));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java
index 7fa6a31..24386fa 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java
@@ -19,6 +19,7 @@ package org.apache.metron.parsers;
import com.google.common.collect.MapDifference;
import com.google.common.collect.Maps;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
@@ -30,6 +31,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
public abstract class GrokParserTest {
@@ -53,8 +55,10 @@ public abstract class GrokParserTest {
JSONObject expected = (JSONObject) jsonParser.parse(e.getValue());
byte[] rawMessage = e.getKey().getBytes();
-
- List<JSONObject> parsedList = grokParser.parse(rawMessage);
+ Optional<MessageParserResult<JSONObject>> resultOptional = grokParser.parseOptionalResult(rawMessage);
+ Assert.assertNotNull(resultOptional);
+ Assert.assertTrue(resultOptional.isPresent());
+ List<JSONObject> parsedList = resultOptional.get().getMessages();
Assert.assertEquals(1, parsedList.size());
compare(expected, parsedList.get(0));
}
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
index cc4d20f..e24a39d 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
@@ -61,7 +61,10 @@ public class MultiLineGrokParserTest {
Map<String, String> testData = getTestData();
for (Map.Entry<String, String> e : testData.entrySet()) {
byte[] rawMessage = e.getKey().getBytes();
- List<JSONObject> parsedList = grokParser.parse(rawMessage);
+ Optional<MessageParserResult<JSONObject>> resultOptional = grokParser.parseOptionalResult(rawMessage);
+ Assert.assertNotNull(resultOptional);
+ Assert.assertTrue(resultOptional.isPresent());
+ List<JSONObject> parsedList = resultOptional.get().getMessages();
Assert.assertEquals(10, parsedList.size());
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
index 0ae2817..e5e7180 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
@@ -17,29 +17,7 @@
*/
package org.apache.metron.parsers.bolt;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.argThat;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
import com.google.common.collect.ImmutableList;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.hbase.util.Bytes;
@@ -55,10 +33,10 @@ import org.apache.metron.common.writer.BulkMessageWriter;
import org.apache.metron.common.writer.BulkWriterResponse;
import org.apache.metron.common.writer.MessageWriter;
import org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater;
-import org.apache.metron.parsers.BasicParser;
import org.apache.metron.parsers.DefaultMessageParserResult;
import org.apache.metron.parsers.interfaces.MessageFilter;
-import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.apache.metron.parsers.interfaces.MultilineMessageParser;
import org.apache.metron.parsers.topology.ParserComponents;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.test.bolt.BaseBoltTest;
@@ -72,10 +50,33 @@ import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mock;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.argThat;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
public class ParserBoltTest extends BaseBoltTest {
@Mock
- private MessageParser<JSONObject> parser;
+ private MultilineMessageParser<JSONObject> parser;
@Mock
private MessageWriter<JSONObject> writer;
@@ -210,7 +211,7 @@ public class ParserBoltTest extends BaseBoltTest {
byte[] sampleBinary = "some binary message".getBytes();
when(tuple.getBinary(0)).thenReturn(sampleBinary);
- when(parser.parseOptional(sampleBinary)).thenReturn(null);
+ when(parser.parseOptionalResult(sampleBinary)).thenReturn(null);
parserBolt.execute(tuple);
verify(parser, times(0)).validate(any());
verify(writer, times(0)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), any());
@@ -405,9 +406,9 @@ public class ParserBoltTest extends BaseBoltTest {
verify(parser, times(1)).init();
verify(batchWriter, times(1)).init(any(), any(), any());
when(parser.validate(any())).thenReturn(true);
- when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject(new HashMap<String, Object>() {{
+ when(parser.parseOptionalResult(any())).thenReturn(Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject(new HashMap<String, Object>() {{
put("field2", "blah");
- }}))));
+ }})))));
parserBolt.execute(t1);
verify(batchWriter, times(0)).write(any(), any(), any(), any());
verify(outputCollector, times(1)).ack(t1);
@@ -437,26 +438,36 @@ public class ParserBoltTest extends BaseBoltTest {
String sensorType = "dummy";
RecordingWriter recordingWriter = new RecordingWriter();
//create a parser which acts like a basic parser but returns no timestamp field.
- BasicParser dummyParser = new BasicParser() {
+ MultilineMessageParser<JSONObject> dummyParser = new MultilineMessageParser<JSONObject>() {
@Override
- public void init() {
-
+ public void configure(Map<String, Object> config) {
}
@Override
- public List<JSONObject> parse(byte[] rawMessage) {
- return ImmutableList.of(new JSONObject() {{
- put("data", "foo");
- put("timestampstr", "2016-01-05 17:02:30");
- put("original_string", "blah");
- }});
+ public void init() {
}
@Override
- public void configure(Map<String, Object> config) {
+ public boolean validate(JSONObject message) {
+ Object timestampObject = message.get(Constants.Fields.TIMESTAMP.getName());
+ if (timestampObject instanceof Long) {
+ Long timestamp = (Long) timestampObject;
+ return timestamp > 0;
+ }
+ return false;
+ }
+ @Override
+ @SuppressWarnings("unchecked")
+ public Optional<MessageParserResult<JSONObject>> parseOptionalResult(byte[] rawMessage) {
+ return Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject() {{
+ put("data", "foo");
+ put("timestampstr", "2016-01-05 17:02:30");
+ put("original_string", "blah");
+ }})));
}
};
+
Map<String, ParserComponents> parserMap = Collections.singletonMap(
sensorType,
new ParserComponents(
@@ -502,7 +513,7 @@ public class ParserBoltTest extends BaseBoltTest {
verify(parser, times(1)).init();
verify(batchWriter, times(1)).init(any(), any(), any());
when(parser.validate(any())).thenReturn(true);
- when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject())));
+ when(parser.parseOptionalResult(any())).thenReturn(Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject()))));
when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
BulkWriterResponse response = new BulkWriterResponse();
Tuple[] uniqueTuples = new Tuple[ParserConfigurations.DEFAULT_KAFKA_BATCH_SIZE];
@@ -592,7 +603,7 @@ public class ParserBoltTest extends BaseBoltTest {
verify(parser, times(1)).init();
verify(batchWriter, times(1)).init(any(), any(), any());
when(parser.validate(any())).thenReturn(true);
- when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject())));
+ when(parser.parseOptionalResult(any())).thenReturn(Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject()))));
when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
BulkWriterResponse response = new BulkWriterResponse();
response.addSuccess(t1);
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
index 0ef26ff..b3e4507 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
@@ -20,6 +20,7 @@ package org.apache.metron.parsers.syslog;
import com.github.palindromicity.syslog.NilPolicy;
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
import org.json.simple.JSONObject;
import org.junit.Assert;
import org.junit.Test;
@@ -28,6 +29,7 @@ import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.function.Consumer;
public class Syslog5424ParserTest {
@@ -87,7 +89,7 @@ public class Syslog5424ParserTest {
});
}
- @Test(expected = IllegalStateException.class)
+ @Test()
public void testNotValid() {
test(null, "not valid", (message) -> Assert.assertTrue(false));
}
@@ -100,7 +102,7 @@ public class Syslog5424ParserTest {
}
parser.configure(config);
- List<JSONObject> output = parser.parse(line.getBytes());
+ parser.parseOptionalResult(line.getBytes());
}
@Test
@@ -116,8 +118,33 @@ public class Syslog5424ParserTest {
.append(SYSLOG_LINE_MISSING)
.append("\n")
.append(SYSLOG_LINE_ALL);
- List<JSONObject> output = parser.parse(builder.toString().getBytes());
- Assert.assertEquals(3,output.size());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(builder.toString().getBytes());
+ Assert.assertNotNull(resultOptional);
+ Assert.assertTrue(resultOptional.isPresent());
+ List<JSONObject> parsedList = resultOptional.get().getMessages();
+ Assert.assertEquals(3,parsedList.size());
+ }
+
+ @Test
+ public void testReadMultiLineWithErrors() throws Exception {
+ Syslog5424Parser parser = new Syslog5424Parser();
+ Map<String, Object> config = new HashMap<>();
+ config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.DASH.name());
+ parser.configure(config);
+ StringBuilder builder = new StringBuilder();
+ builder
+ .append("HEREWEGO!!!!\n")
+ .append(SYSLOG_LINE_ALL)
+ .append("\n")
+ .append(SYSLOG_LINE_MISSING)
+ .append("\n")
+ .append("BOOM!\n")
+ .append(SYSLOG_LINE_ALL)
+ .append("\nOHMY!");
+ Optional<MessageParserResult<JSONObject>> output = parser.parseOptionalResult(builder.toString().getBytes());
+ Assert.assertTrue(output.isPresent());
+ Assert.assertEquals(3,output.get().getMessages().size());
+ Assert.assertEquals(3,output.get().getMessageThrowables().size());
}
@Test
@@ -126,21 +153,29 @@ public class Syslog5424ParserTest {
Map<String, Object> config = new HashMap<>();
config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.DASH.name());
parser.configure(config);
- List<JSONObject> output = parser.parse(SYSLOG_LINE_MISSING_DATE.getBytes());
- String timeStampString = output.get(0).get("timestamp").toString();
+ Optional<MessageParserResult<JSONObject>> output = parser.parseOptionalResult(SYSLOG_LINE_MISSING_DATE.getBytes());
+ Assert.assertNotNull(output);
+ Assert.assertTrue(output.isPresent());
+ String timeStampString = output.get().getMessages().get(0).get("timestamp").toString();
DateTimeFormatter.ISO_DATE_TIME.parse(timeStampString);
config.clear();
config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.NULL.name());
parser.configure(config);
- output = parser.parse(SYSLOG_LINE_MISSING_DATE.getBytes());
- timeStampString = output.get(0).get("timestamp").toString();
+ output = parser.parseOptionalResult(SYSLOG_LINE_MISSING_DATE.getBytes());
+ Assert.assertNotNull(output);
+ Assert.assertTrue(output.isPresent());
+ timeStampString = output.get().getMessages().get(0).get("timestamp").toString();
DateTimeFormatter.ISO_DATE_TIME.parse(timeStampString);
config.clear();
config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.OMIT.name());
parser.configure(config);
- output = parser.parse(SYSLOG_LINE_MISSING_DATE.getBytes());
- timeStampString = output.get(0).get("timestamp").toString();
+
+ output = parser.parseOptionalResult(SYSLOG_LINE_MISSING_DATE.getBytes());
+ Assert.assertNotNull(output);
+ Assert.assertTrue(output.isPresent());
+
+ timeStampString = output.get().getMessages().get(0).get("timestamp").toString();
DateTimeFormatter.ISO_DATE_TIME.parse(timeStampString);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
index 2923a4f..eb447d0 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
@@ -24,11 +24,14 @@ import java.time.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import org.apache.log4j.Level;
import org.apache.metron.parsers.GrokParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
import org.apache.metron.test.utils.UnitTestHelper;
import org.json.simple.JSONObject;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -54,7 +57,10 @@ public class GrokWebSphereParserTest {
parser.configure(parserConfig);
String testString = "<133>Apr 15 17:47:28 ABCXML1413 [rojOut][0x81000033][auth][notice] user(rick007): "
+ "[120.43.200.6]: User logged into 'cohlOut'.";
- List<JSONObject> result = parser.parse(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Assert.assertNotNull(resultOptional);
+ Assert.assertTrue(resultOptional.isPresent());
+ List<JSONObject> result = resultOptional.get().getMessages();
JSONObject parsedJSON = result.get(0);
long expectedTimestamp = ZonedDateTime.of(Year.now(UTC).getValue(), 4, 15, 17, 47, 28, 0, UTC).toInstant().toEpochMilli();
@@ -73,14 +79,17 @@ public class GrokWebSphereParserTest {
}
@Test
- public void tetsParseLogoutLine() throws Exception {
+ public void testParseLogoutLine() throws Exception {
//Set up parser, parse message
GrokWebSphereParser parser = new GrokWebSphereParser();
parser.configure(parserConfig);
String testString = "<134>Apr 15 18:02:27 PHIXML3RWD [0x81000019][auth][info] [14.122.2.201]: "
+ "User 'hjpotter' logged out from 'default'.";
- List<JSONObject> result = parser.parse(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Assert.assertNotNull(resultOptional);
+ Assert.assertTrue(resultOptional.isPresent());
+ List<JSONObject> result = resultOptional.get().getMessages();
JSONObject parsedJSON = result.get(0);
long expectedTimestamp = ZonedDateTime.of(Year.now(UTC).getValue(), 4, 15, 18, 2, 27, 0, UTC).toInstant().toEpochMilli();
@@ -98,14 +107,17 @@ public class GrokWebSphereParserTest {
}
@Test
- public void tetsParseRBMLine() throws Exception {
+ public void testParseRBMLine() throws Exception {
//Set up parser, parse message
GrokWebSphereParser parser = new GrokWebSphereParser();
parser.configure(parserConfig);
String testString = "<131>Apr 15 17:36:35 ROBXML3QRS [0x80800018][auth][error] rbm(RBM-Settings): "
+ "trans(3502888135)[request] gtid(3502888135): RBM: Resource access denied.";
- List<JSONObject> result = parser.parse(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Assert.assertNotNull(resultOptional);
+ Assert.assertTrue(resultOptional.isPresent());
+ List<JSONObject> result = resultOptional.get().getMessages();
JSONObject parsedJSON = result.get(0);
long expectedTimestamp = ZonedDateTime.of(Year.now(UTC).getValue(), 4, 15, 17, 36, 35, 0, UTC).toInstant().toEpochMilli();
@@ -122,16 +134,18 @@ public class GrokWebSphereParserTest {
}
@Test
- public void tetsParseOtherLine() throws Exception {
+ public void testParseOtherLine() throws Exception {
//Set up parser, parse message
GrokWebSphereParser parser = new GrokWebSphereParser();
parser.configure(parserConfig);
String testString = "<134>Apr 15 17:17:34 SAGPXMLQA333 [0x8240001c][audit][info] trans(191): (admin:default:system:*): "
+ "ntp-service 'NTP Service' - Operational state down";
- List<JSONObject> result = parser.parse(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Assert.assertNotNull(resultOptional);
+ Assert.assertTrue(resultOptional.isPresent());
+ List<JSONObject> result = resultOptional.get().getMessages();
JSONObject parsedJSON = result.get(0);
-
long expectedTimestamp = ZonedDateTime.of(Year.now(UTC).getValue(), 4, 15, 17, 17, 34, 0, UTC).toInstant().toEpochMilli();
//Compare fields
@@ -153,7 +167,10 @@ public class GrokWebSphereParserTest {
parser.configure(parserConfig);
String testString = "<133>Apr 15 17:47:28 ABCXML1413 [rojOut][0x81000033][auth][notice] rick007): "
+ "[120.43.200. User logged into 'cohlOut'.";
- List<JSONObject> result = parser.parse(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Assert.assertNotNull(resultOptional);
+ Assert.assertTrue(resultOptional.isPresent());
+ List<JSONObject> result = resultOptional.get().getMessages();
JSONObject parsedJSON = result.get(0);
long expectedTimestamp = ZonedDateTime.of(Year.now(UTC).getValue(), 4, 15, 17, 47, 28, 0, UTC).toInstant().toEpochMilli();
@@ -172,14 +189,17 @@ public class GrokWebSphereParserTest {
}
@Test
- public void tetsParseMalformedLogoutLine() throws Exception {
+ public void testParseMalformedLogoutLine() throws Exception {
//Set up parser, attempt to parse malformed message
GrokWebSphereParser parser = new GrokWebSphereParser();
parser.configure(parserConfig);
String testString = "<134>Apr 15 18:02:27 PHIXML3RWD [0x81000019][auth][info] [14.122.2.201: "
+ "User 'hjpotter' logged out from 'default.";
- List<JSONObject> result = parser.parse(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Assert.assertNotNull(resultOptional);
+ Assert.assertTrue(resultOptional.isPresent());
+ List<JSONObject> result = resultOptional.get().getMessages();
JSONObject parsedJSON = result.get(0);
long expectedTimestamp = ZonedDateTime.of(Year.now(UTC).getValue(), 4, 15, 18, 2, 27, 0, UTC).toInstant().toEpochMilli();
@@ -197,14 +217,17 @@ public class GrokWebSphereParserTest {
}
@Test
- public void tetsParseMalformedRBMLine() throws Exception {
+ public void testParseMalformedRBMLine() throws Exception {
//Set up parser, parse message
GrokWebSphereParser parser = new GrokWebSphereParser();
parser.configure(parserConfig);
String testString = "<131>Apr 15 17:36:35 ROBXML3QRS [0x80800018][auth][error] rbmRBM-Settings): "
+ "trans3502888135)[request] gtid3502888135) RBM: Resource access denied.";
- List<JSONObject> result = parser.parse(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Assert.assertNotNull(resultOptional);
+ Assert.assertTrue(resultOptional.isPresent());
+ List<JSONObject> result = resultOptional.get().getMessages();
JSONObject parsedJSON = result.get(0);
long expectedTimestamp = ZonedDateTime.of(Year.now(UTC).getValue(), 4, 15, 17, 36, 35, 0, UTC).toInstant().toEpochMilli();
@@ -221,14 +244,17 @@ public class GrokWebSphereParserTest {
}
@Test
- public void tetsParseMalformedOtherLine() throws Exception {
+ public void testParseMalformedOtherLine() throws Exception {
//Set up parser, parse message
GrokWebSphereParser parser = new GrokWebSphereParser();
parser.configure(parserConfig);
String testString = "<134>Apr 15 17:17:34 SAGPXMLQA333 [0x8240001c][audit][info] trans 191) admindefaultsystem*): "
+ "ntp-service 'NTP Service' - Operational state down:";
- List<JSONObject> result = parser.parse(testString.getBytes());
+ Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes());
+ Assert.assertNotNull(resultOptional);
+ Assert.assertTrue(resultOptional.isPresent());
+ List<JSONObject> result = resultOptional.get().getMessages();
JSONObject parsedJSON = result.get(0);
long expectedTimestamp = ZonedDateTime.of(Year.now(UTC).getValue(), 4, 15, 17, 17, 34, 0, UTC).toInstant().toEpochMilli();
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1e6adb0..f412036 100644
--- a/pom.xml
+++ b/pom.xml
@@ -137,7 +137,7 @@
<global_reflections_version>0.9.10</global_reflections_version>
<global_checkstyle_version>8.0</global_checkstyle_version>
<global_log4j_core_version>2.1</global_log4j_core_version>
- <global_simple_syslog_version>0.0.8</global_simple_syslog_version>
+ <global_simple_syslog_version>0.0.9</global_simple_syslog_version>
<global_spark_version>2.3.1</global_spark_version>
</properties>
[27/51] [abbrv] metron git commit: METRON-1695 Expose pcap properties
through Ambari (anandsubbu) closes apache/metron#1207
Posted by ot...@apache.org.
METRON-1695 Expose pcap properties through Ambari (anandsubbu) closes apache/metron#1207
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/feb9153a
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/feb9153a
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/feb9153a
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: feb9153a610e7cc384055c5d07b5c1dcf8b81923
Parents: de53306
Author: anandsubbu <an...@gmail.com>
Authored: Fri Oct 5 12:09:55 2018 +0530
Committer: anandsubbu <an...@apache.org>
Committed: Fri Oct 5 12:09:55 2018 +0530
----------------------------------------------------------------------
.../roles/ambari_config/vars/single_node_vm.yml | 2 +-
metron-deployment/packaging/ambari/.gitignore | 1 +
.../packaging/ambari/metron-mpack/pom.xml | 7 +
.../CURRENT/configuration/metron-pcap-env.xml | 188 +++++++++++++++++++
.../CURRENT/configuration/metron-rest-env.xml | 47 -----
.../common-services/METRON/CURRENT/metainfo.xml | 3 +-
.../package/scripts/params/params_linux.py | 33 +++-
.../package/scripts/params/status_params.py | 2 +-
.../CURRENT/package/scripts/pcap_master.py | 7 +
.../METRON/CURRENT/themes/metron_theme.json | 174 +++++++++++++++--
metron-interface/metron-rest/README.md | 4 +-
metron-platform/metron-pcap-backend/README.md | 3 +
.../src/main/assembly/assembly.xml | 1 +
.../src/main/config/pcap.properties | 9 +-
.../src/main/config/pcap.properties.j2 | 40 ++++
15 files changed, 446 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index 2ec956a..f67522c 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -105,7 +105,7 @@ configurations:
- metron-rest-env:
metron_spring_profiles_active: "dev"
- metron-parsers-env:
- parsers: "bro,snort"
+ parsers: "\"bro,snort,yaf\""
- elastic-site:
index_number_of_shards: 1
index_number_of_replicas: 0
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/.gitignore b/metron-deployment/packaging/ambari/.gitignore
index d2cc43c..242a4da 100644
--- a/metron-deployment/packaging/ambari/.gitignore
+++ b/metron-deployment/packaging/ambari/.gitignore
@@ -6,3 +6,4 @@ hdfs.properties.j2
enrichment.properties.j2
enrichment-splitjoin.properties.j2
enrichment-unified.properties.j2
+pcap.properties.j2
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
index 82386c1..2975963 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -136,6 +136,13 @@
</includes>
<filtering>false</filtering>
</resource>
+ <resource>
+ <directory>${basedir}/../../../../metron-platform/metron-pcap-backend/src/main/config</directory>
+ <includes>
+ <include>pcap.properties.j2</include>
+ </includes>
+ <filtering>false</filtering>
+ </resource>
</resources>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml
new file mode 100644
index 0000000..31f9e8f
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml
@@ -0,0 +1,188 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>pcap_topology_worker_childopts</name>
+ <description>PCAP Topology JVM Options</description>
+ <value/>
+ <display-name>PCAP Topology childopts</display-name>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>pcap_topology_workers</name>
+ <description>Number of PCAP Topology Workers</description>
+ <value>1</value>
+ <display-name>Workers for PCAP Topology</display-name>
+ </property>
+ <property>
+ <name>spout_kafka_topic_pcap</name>
+ <description>PCAP Input Topic</description>
+ <value>pcap</value>
+ <display-name>PCAP Input Topic</display-name>
+ </property>
+ <property>
+ <name>hdfs_sync_every</name>
+ <description>HDFS Sync Every</description>
+ <value>1</value>
+ <display-name>HDFS Sync Every</display-name>
+ </property>
+ <property>
+ <name>hdfs_replication_factor</name>
+ <description>HDFS Replication Factor</description>
+ <value>-1</value>
+ <display-name>HDFS Replication Factor</display-name>
+ </property>
+ <property>
+ <name>kafka_pcap_start</name>
+ <description>PCAP Topology Spout Offset</description>
+ <value>UNCOMMITTED_EARLIEST</value>
+ <display-name>PCAP Topology Offset</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>EARLIEST</value>
+ </entry>
+ <entry>
+ <value>LATEST</value>
+ </entry>
+ <entry>
+ <value>UNCOMMITTED_EARLIEST</value>
+ </entry>
+ <entry>
+ <value>UNCOMMITTED_LATEST</value>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+ <property>
+ <name>kafka_pcap_numpackets</name>
+ <description>Number of Packets</description>
+ <value>1000</value>
+ <display-name>Number of Packets to keep in one file</display-name>
+ </property>
+ <property>
+ <name>kafka_pcap_maxtimems</name>
+ <description>Max Time in milliseconds</description>
+ <value>300000</value>
+ <display-name>Number of packets to keep in terms of duration</display-name>
+ </property>
+ <property>
+ <name>kafka_pcap_tsscheme</name>
+ <description>Kafka PCAP Timestamp Scheme. Use FROM_KEY which works with current tooling</description>
+ <value>FROM_KEY</value>
+ <display-name>Kafka PCAP Timestamp Scheme</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>FROM_KEY</value>
+ </entry>
+ <entry>
+ <value>FROM_VALUE</value>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+ <property>
+ <name>kafka_pcap_out</name>
+ <description>HDFS Directory to store PCAPs</description>
+ <value>/apps/metron/pcap/input</value>
+ <display-name>HDFS Directory to store PCAPs</display-name>
+ </property>
+ <property>
+ <name>kafka_pcap_ts_granularity</name>
+ <description>Granulariy of Timing in Timestamps</description>
+ <value>MICROSECONDS</value>
+ <display-name>Granulariy of Timing in Timestamps</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>MICROSECONDS</value>
+ </entry>
+ <entry>
+ <value>MILLISECONDS</value>
+ </entry>
+ <entry>
+ <value>NANOSECONDS</value>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+ <property>
+ <name>kafka_spout_parallelism</name>
+ <description>PCAP Topology Spout Parallelism</description>
+ <value>1</value>
+ <display-name>PCAP Topology Spout Parallelism</display-name>
+ </property>
+ <property>
+ <name>pcap_base_path</name>
+ <display-name>PCAP Base Path</display-name>
+ <description>The HDFS path where pcap files are stored. These files are the input to pcap queries.</description>
+ <value>/apps/metron/pcap/input</value>
+ </property>
+ <property>
+ <name>pcap_base_interim_result_path</name>
+ <display-name>PCAP Base Interim Result Path</display-name>
+ <description>The HDFS path where interim results of a pcap query are stored, before they are separated into pages.</description>
+ <value>/apps/metron/pcap/interim</value>
+ </property>
+ <property>
+ <name>pcap_final_output_path</name>
+ <display-name>PCAP Final Output Path</display-name>
+ <description>The HDFS path where the final results of a pcap query are stored.</description>
+ <value>/apps/metron/pcap/output</value>
+ </property>
+ <property>
+ <name>pcap_page_size</name>
+ <display-name>PCAP Page Size</display-name>
+ <description>The number of pcaps written to a page/file as a result of a pcap query.</description>
+ <value>10</value>
+ </property>
+ <property>
+ <name>pcap_yarn_queue</name>
+ <display-name>PCAP YARN Queue</display-name>
+ <description>The YARN queue pcap jobs will be submitted to.</description>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>pcap_finalizer_threadpool_size</name>
+ <display-name>PCAP Finalizer Threadpool Size</display-name>
+ <description>The number of threads to use when finalizing Pcap jobs. This affects parallelism
+ around writing out paged files to their final location.
+ If it's a string and ends with "C", then strip the C and treat it as an integral multiple of
+ the number of cores. If it's a string and does not end with a C, then treat it as a number in
+ string form.
+ </description>
+ <value>1</value>
+ <value-attributes>
+ <empty-value-valid>false</empty-value-valid>
+ </value-attributes>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
index 767afa3..f4b2327 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -168,52 +168,5 @@
<description>The field name where the threat triage score can be found in the search indices. This setting primarily affects the Alerts UI.</description>
<value>threat:triage:score</value>
</property>
- <property>
- <name>pcap_base_path</name>
- <display-name>Pcap Base Path</display-name>
- <description>The HDFS path where pcap files are stored. These files are the input to pcap queries.</description>
- <value>/apps/metron/pcap/input</value>
- </property>
- <property>
- <name>pcap_base_interim_result_path</name>
- <display-name>Pcap Base Interim Result Path</display-name>
- <description>The HDFS path where interim results of a pcap query are stored, before they are separated into pages.</description>
- <value>/apps/metron/pcap/interim</value>
- </property>
- <property>
- <name>pcap_final_output_path</name>
- <display-name>Pcap Final Output Path</display-name>
- <description>The HDFS path where the final results of a pcap query are stored.</description>
- <value>/apps/metron/pcap/output</value>
- </property>
- <property>
- <name>pcap_page_size</name>
- <display-name>Pcap Page Size</display-name>
- <description>The number of pcaps written to a page/file as a result of a pcap query.</description>
- <value>10</value>
- </property>
- <property>
- <name>pcap_yarn_queue</name>
- <display-name>Pcap YARN Queue</display-name>
- <description>The YARN queue pcap jobs will be submitted to.</description>
- <value/>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- </property>
- <property>
- <name>pcap_finalizer_threadpool_size</name>
- <display-name>Pcap Finalizer Threadpool Size</display-name>
- <description>The number of threads to use when finalizing Pcap jobs. This affects parallelism
- around writing out paged files to their final location.
- If it's a string and ends with "C", then strip the C and treat it as an integral multiple of
- the number of cores. If it's a string and does not end with a C, then treat it as a number in
- string form.
- </description>
- <value>1</value>
- <value-attributes>
- <empty-value-valid>false</empty-value-valid>
- </value-attributes>
- </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 3c8a4ea..ad1f7a9 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -193,7 +193,7 @@
<timeout>600</timeout>
</commandScript>
<configuration-dependencies>
- <config-type>metron-rest-env</config-type>
+ <config-type>metron-pcap-env</config-type>
</configuration-dependencies>
</component>
@@ -324,6 +324,7 @@
<configuration-dependencies>
<config-type>metron-indexing-env</config-type>
<config-type>metron-rest-env</config-type>
+ <config-type>metron-pcap-env</config-type>
</configuration-dependencies>
</component>
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index dbad44d..458a7be 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -241,6 +241,7 @@ client_jaas_arg = '-Djava.security.auth.login.config=' + metron_home + '/client_
enrichment_topology_worker_childopts = client_jaas_arg if security_enabled else ''
profiler_topology_worker_childopts = client_jaas_arg if security_enabled else ''
indexing_topology_worker_childopts = client_jaas_arg if security_enabled else ''
+pcap_topology_worker_childopts = client_jaas_arg if security_enabled else ''
metron_jvm_flags += (' ' + client_jaas_arg) if security_enabled else ''
topology_auto_credentials = config['configurations']['storm-site'].get('nimbus.credential.renewers.classes', [])
# Needed for storm.config, because it needs Java String
@@ -387,18 +388,32 @@ bolt_hdfs_rotation_policy = config['configurations']['metron-indexing-env']['bol
bolt_hdfs_rotation_policy_units = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_units']
bolt_hdfs_rotation_policy_count = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count']
-# Pcap
-metron_pcap_topology = 'pcap'
-pcap_input_topic = 'pcap'
-pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path']
-pcap_base_interim_result_path = config['configurations']['metron-rest-env']['pcap_base_interim_result_path']
-pcap_final_output_path = config['configurations']['metron-rest-env']['pcap_final_output_path']
-pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size']
-pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue']
-pcap_finalizer_threadpool_size= config['configurations']['metron-rest-env']['pcap_finalizer_threadpool_size']
+# PCAP
+metron_pcap_topology = status_params.metron_pcap_topology
+pcap_input_topic = status_params.pcap_input_topic
+pcap_base_path = config['configurations']['metron-pcap-env']['pcap_base_path']
+pcap_base_interim_result_path = config['configurations']['metron-pcap-env']['pcap_base_interim_result_path']
+pcap_final_output_path = config['configurations']['metron-pcap-env']['pcap_final_output_path']
+pcap_page_size = config['configurations']['metron-pcap-env']['pcap_page_size']
+pcap_yarn_queue = config['configurations']['metron-pcap-env']['pcap_yarn_queue']
+pcap_finalizer_threadpool_size= config['configurations']['metron-pcap-env']['pcap_finalizer_threadpool_size']
pcap_configured_flag_file = status_params.pcap_configured_flag_file
pcap_perm_configured_flag_file = status_params.pcap_perm_configured_flag_file
pcap_acl_configured_flag_file = status_params.pcap_acl_configured_flag_file
+pcap_topology_workers = config['configurations']['metron-pcap-env']['pcap_topology_workers']
+if not len(pcap_topology_worker_childopts) == 0:
+ pcap_topology_worker_childopts += ' '
+pcap_topology_worker_childopts += config['configurations']['metron-pcap-env']['pcap_topology_worker_childopts']
+spout_kafka_topic_pcap = config['configurations']['metron-pcap-env']['spout_kafka_topic_pcap']
+hdfs_sync_every = config['configurations']['metron-pcap-env']['hdfs_sync_every']
+hdfs_replication_factor = config['configurations']['metron-pcap-env']['hdfs_replication_factor']
+kafka_pcap_start = config['configurations']['metron-pcap-env']['kafka_pcap_start']
+kafka_pcap_numpackets = config['configurations']['metron-pcap-env']['kafka_pcap_numpackets']
+kafka_pcap_maxtimems = config['configurations']['metron-pcap-env']['kafka_pcap_maxtimems']
+kafka_pcap_tsscheme = config['configurations']['metron-pcap-env']['kafka_pcap_tsscheme']
+kafka_pcap_out = config['configurations']['metron-pcap-env']['kafka_pcap_out']
+kafka_pcap_ts_granularity = config['configurations']['metron-pcap-env']['kafka_pcap_ts_granularity']
+kafka_spout_parallelism = config['configurations']['metron-pcap-env']['kafka_spout_parallelism']
# MapReduce
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 1ca29b2..aad34a9 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -120,7 +120,7 @@ metron_keytab_path = config['configurations']['metron-env']['metron_service_keyt
# Pcap
metron_pcap_topology = 'pcap'
-pcap_input_topic = 'pcap'
+pcap_input_topic = config['configurations']['metron-pcap-env']['spout_kafka_topic_pcap']
pcap_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_configured'
pcap_perm_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_perm_configured'
pcap_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_acl_configured'
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
index a3bc1b4..109c035 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
@@ -50,6 +50,13 @@ class Pcap(Script):
commands = PcapCommands(params)
+ Logger.info("Running PCAP configure")
+ File(format("{metron_config_path}/pcap.properties"),
+ content=Template("pcap.properties.j2"),
+ owner=params.metron_user,
+ group=params.metron_group
+ )
+
if not commands.is_configured():
commands.init_kafka_topics()
commands.init_hdfs_dir()
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index 2b64f8f..7e6c83a 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -420,6 +420,34 @@
}
]
}
+ },
+ {
+ "name": "metron-pcap",
+ "display-name": "PCAP",
+ "layout": {
+ "tab-columns": "1",
+ "tab-rows": "1",
+ "sections": [
+ {
+ "name": "section-pcap",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-pcap",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
}
]
}
@@ -820,24 +848,72 @@
"subsection-name": "subsection-rest"
},
{
- "config": "metron-rest-env/pcap_page_size",
- "subsection-name": "subsection-rest"
+ "config": "metron-management-ui-env/metron_management_ui_port",
+ "subsection-name": "subsection-management-ui"
},
{
- "config": "metron-rest-env/pcap_yarn_queue",
- "subsection-name": "subsection-rest"
+ "config": "metron-alerts-ui-env/metron_alerts_ui_port",
+ "subsection-name": "subsection-alerts-ui"
},
{
- "config": "metron-rest-env/pcap_finalizer_threadpool_size",
- "subsection-name": "subsection-rest"
+ "config": "metron-pcap-env/pcap_topology_workers",
+ "subsection-name": "subsection-pcap"
},
{
- "config": "metron-management-ui-env/metron_management_ui_port",
- "subsection-name": "subsection-management-ui"
+ "config": "metron-pcap-env/pcap_topology_worker_childopts",
+ "subsection-name": "subsection-pcap"
},
{
- "config": "metron-alerts-ui-env/metron_alerts_ui_port",
- "subsection-name": "subsection-alerts-ui"
+ "config": "metron-pcap-env/spout_kafka_topic_pcap",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/hdfs_sync_every",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/hdfs_replication_factor",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_start",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_numpackets",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_maxtimems",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_tsscheme",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_out",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_ts_granularity",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/kafka_spout_parallelism",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/pcap_page_size",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/pcap_yarn_queue",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/pcap_finalizer_threadpool_size",
+ "subsection-name": "subsection-pcap"
}
]
},
@@ -1433,19 +1509,19 @@
}
},
{
- "config": "metron-rest-env/pcap_page_size",
+ "config": "metron-pcap-env/pcap_page_size",
"widget": {
"type": "text-field"
}
},
{
- "config": "metron-rest-env/pcap_yarn_queue",
+ "config": "metron-pcap-env/pcap_yarn_queue",
"widget": {
"type": "text-field"
}
},
{
- "config": "metron-rest-env/pcap_finalizer_threadpool_size",
+ "config": "metron-pcap-env/pcap_finalizer_threadpool_size",
"widget": {
"type": "text-field"
}
@@ -1461,6 +1537,78 @@
"widget": {
"type": "text-field"
}
+ },
+ {
+ "config": "metron-pcap-env/pcap_topology_workers",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/pcap_topology_worker_childopts",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/spout_kafka_topic_pcap",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/hdfs_sync_every",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/hdfs_replication_factor",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_start",
+ "widget": {
+ "type": "combo"
+ }
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_numpackets",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_maxtimems",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_tsscheme",
+ "widget": {
+ "type": "combo"
+ }
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_out",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_ts_granularity",
+ "widget": {
+ "type": "combo"
+ }
+ },
+ {
+ "config": "metron-pcap-env/kafka_spout_parallelism",
+ "widget": {
+ "type": "text-field"
+ }
}
]
}
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index 2ce9522..7f00cde 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -221,7 +221,7 @@ The REST application uses a Java Process object to call out to the `pcap_to_pdml
Out of the box it is a simple wrapper around the tshark command to transform raw pcap data to PDML. However it can be extended to do additional processing as long as the expected input/output is maintained.
REST will supply the script with raw pcap data through standard in and expects PDML data serialized as XML.
-Pcap query jobs can be configured for submission to a YARN queue. This setting is exposed as the Spring property `pcap.yarn.queue`. If configured, the REST application will set the `mapreduce.job.queuename` Hadoop property to that value.
+Pcap query jobs can be configured for submission to a YARN queue. This setting is exposed as the Spring property `pcap.yarn.queue` and can be set in the PCAP tab under Metron service -> Configs in Ambari. If configured, the REST application will set the `mapreduce.job.queuename` Hadoop property to that value.
It is highly recommended that a dedicated YARN queue be created and configured for Pcap queries to prevent a job from consuming too many cluster resources. More information about setting up YARN queues can be found [here](https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html#Setting_up_queues).
Pcap query results are stored in HDFS. The location of query results when run through the REST app is determined by a couple factors. The root of Pcap query results defaults to `/apps/metron/pcap/output` but can be changed with the
@@ -234,7 +234,7 @@ periodically delete files and directories under the Pcap query results root.
Users should also be mindful of date ranges used in queries so they don't produce result sets that are too large. Currently there are no limits enforced on date ranges.
-Queries can also be configured on a global level for setting the number of results per page via a Spring property `pcap.page.size`. By default, this value is set to 10 pcaps per page, but you may choose to set this value higher
+Queries can also be configured on a global level for setting the number of results per page via a Spring property `pcap.page.size`. This property can be set in the PCAP tab under Metron service -> Configs, in Ambari. By default, this value is set to 10 pcaps per page, but you may choose to set this value higher
based on observing frequenetly-run query result sizes. This setting works in conjunction with the property for setting finalizer threadpool size when optimizing query performance.
Pcap query jobs have a finalization routine that writes their results out to HDFS in pages. Depending on the size of your pcaps, the number or results typically returned, page sizing (described above), and available CPU cores for running
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-platform/metron-pcap-backend/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/README.md b/metron-platform/metron-pcap-backend/README.md
index 031328d..49063e5 100644
--- a/metron-platform/metron-pcap-backend/README.md
+++ b/metron-platform/metron-pcap-backend/README.md
@@ -70,6 +70,9 @@ sequence files.
## Configuration
+The configuration properties for PCAP sensor is managed via Ambari at Services -> Metron -> Config -> PCAP tab.
+Note that changes to PCAP sensor config properties via Ambari requires restarting the Metron PCAP service.
+
The configuration file for the Flux topology is located at
`$METRON_HOME/config/pcap.properties` and the possible options
are as follows:
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml b/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml
index 5b21e10..f38582d 100644
--- a/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml
@@ -25,6 +25,7 @@
<excludes>
<exclude>**/*.formatted</exclude>
<exclude>**/*.filtered</exclude>
+ <exclude>**/*.j2</exclude>
</excludes>
<fileMode>0644</fileMode>
<lineEnding>unix</lineEnding>
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties b/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
index 848b588..7ee8a95 100644
--- a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
+++ b/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
@@ -14,19 +14,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-spout.kafka.topic.pcap=pcap
+##### Storm #####
topology.worker.childopts=
topology.auto-credentials=[]
topology.workers=1
+
+##### Kafka #####
+spout.kafka.topic.pcap=pcap
kafka.zk=node1:2181
hdfs.sync.every=1
hdfs.replication.factor=-1
kafka.security.protocol=PLAINTEXT
+
# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
kafka.pcap.start=UNCOMMITTED_EARLIEST
+
kafka.pcap.numPackets=1000
kafka.pcap.maxTimeMS=300000
kafka.pcap.ts_scheme=FROM_KEY
kafka.pcap.out=/apps/metron/pcap/input
+
+##### Parallelism #####
kafka.pcap.ts_granularity=MICROSECONDS
kafka.spout.parallelism=1
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties.j2
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties.j2 b/metron-platform/metron-pcap-backend/src/main/config/pcap.properties.j2
new file mode 100644
index 0000000..21f676a
--- /dev/null
+++ b/metron-platform/metron-pcap-backend/src/main/config/pcap.properties.j2
@@ -0,0 +1,40 @@
+{# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+##### Storm #####
+topology.worker.childopts={{pcap_topology_worker_childopts}}
+topology.auto-credentials={{topology_auto_credentials}}
+topology.workers={{pcap_topology_workers}}
+
+##### Kafka #####
+spout.kafka.topic.pcap={{spout_kafka_topic_pcap}}
+kafka.zk={{zookeeper_quorum}}
+hdfs.sync.every={{hdfs_sync_every}}
+hdfs.replication.factor={{hdfs_replication_factor}}
+kafka.security.protocol={{kafka_security_protocol}}
+
+# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
+kafka.pcap.start={{kafka_pcap_start}}
+
+kafka.pcap.numPackets={{kafka_pcap_numpackets}}
+kafka.pcap.maxTimeMS={{kafka_pcap_maxtimems}}
+kafka.pcap.ts_scheme={{kafka_pcap_tsscheme}}
+kafka.pcap.out={{kafka_pcap_out}}
+
+##### Parallelism #####
+kafka.pcap.ts_granularity={{kafka_pcap_ts_granularity}}
+kafka.spout.parallelism={{kafka_spout_parallelism}}
\ No newline at end of file
[36/51] [abbrv] metron git commit: METRON-1761 Allow a grok statement
to be applied to each line in a file. (ottobackwards) closes
apache/metron#1184
Posted by ot...@apache.org.
METRON-1761 Allow a grok statement to be applied to each line in a file. (ottobackwards) closes apache/metron#1184
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3d923cde
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3d923cde
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3d923cde
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 3d923cde8711dcc463c2d27cb2b67275b9172112
Parents: 060d17e
Author: ottobackwards <ot...@gmail.com>
Authored: Wed Oct 10 11:31:31 2018 -0400
Committer: otto <ot...@apache.org>
Committed: Wed Oct 10 11:31:31 2018 -0400
----------------------------------------------------------------------
.../src/main/sample/patterns/test | 1 +
metron-platform/metron-parsers/README.md | 10 +-
.../message-parser-implementation-notes.md | 57 +++++
.../parsers/DefaultMessageParserResult.java | 76 +++++++
.../org/apache/metron/parsers/GrokParser.java | 156 ++++++++++---
.../apache/metron/parsers/bolt/ParserBolt.java | 57 +++--
.../parsers/interfaces/MessageParser.java | 28 ++-
.../parsers/interfaces/MessageParserResult.java | 48 ++++
.../parsers/websphere/GrokWebSphereParser.java | 217 +++++++++----------
.../apache/metron/parsers/GrokParserTest.java | 1 +
.../metron/parsers/MultiLineGrokParserTest.java | 146 +++++++++++++
.../MultiLineWithErrorsGrokParserTest.java | 146 +++++++++++++
.../metron/parsers/SampleGrokParserTest.java | 3 +
.../apache/metron/parsers/SquidParserTest.java | 2 +
.../apache/metron/parsers/YafParserTest.java | 3 +
.../metron/parsers/bolt/ParserBoltTest.java | 15 +-
.../parsers/integration/ParserDriver.java | 2 +-
.../websphere/GrokWebSphereParserTest.java | 13 --
.../test/resources/logData/multi_elb_log.txt | 10 +
.../logData/multi_elb_with_errors_log.txt | 13 ++
20 files changed, 824 insertions(+), 180 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-integration-test/src/main/sample/patterns/test
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/sample/patterns/test b/metron-platform/metron-integration-test/src/main/sample/patterns/test
index a88a255..ebbf9c4 100644
--- a/metron-platform/metron-integration-test/src/main/sample/patterns/test
+++ b/metron-platform/metron-integration-test/src/main/sample/patterns/test
@@ -1,2 +1,3 @@
YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T ]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED}
YAF_DELIMITED %{NUMBER:start_time}\|%{YAF_TIME_FORMAT:end_time}\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\|%{SPACE:UNWANTED}%{INT:protocol}\|%{SPACE:UNWANTED}%{IP:ip_src_addr}\|%{SPACE:UNWANTED}%{INT:ip_src_port}\|%{SPACE:UNWANTED}%{IP:ip_dst_addr}\|%{SPACE:UNWANTED}%{INT:ip_dst_port}\|%{SPACE:UNWANTED}%{DATA:iflags}\|%{SPACE:UNWANTED}%{DATA:uflags}\|%{SPACE:UNWANTED}%{DATA:riflags}\|%{SPACE:UNWANTED}%{DATA:ruflags}\|%{SPACE:UNWANTED}%{WORD:isn}\|%{SPACE:UNWANTED}%{DATA:risn}\|%{SPACE:UNWANTED}%{DATA:tag}\|%{GREEDYDATA:rtag}\|%{SPACE:UNWANTED}%{INT:pkt}\|%{SPACE:UNWANTED}%{INT:oct}\|%{SPACE:UNWANTED}%{INT:rpkt}\|%{SPACE:UNWANTED}%{INT:roct}\|%{SPACE:UNWANTED}%{INT:app}\|%{GREEDYDATA:end_reason}
+ELBACCESSLOGS %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport} (?:(%{IP:backendip}:?:%{INT:backendport})|-) %{NUMBER:request_processing_time} %{NUMBER:backend_processing_time} %{NUMBER:response_processing_time} (?:-|%{INT:elb_status_code}) (?:-|%{INT:backend_status_code}) %{INT:received_bytes} %{INT:sent_bytes} \"(?:-|(?:%{WORD:verb} %{URIPROTO:proto}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST:urihost})?(?:%{URIPATH:path}(?:%{URIPARAM:params})?)?(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest}))\" \"(?:-|%{DATA:user_agent})\" (?:-|%{NOTSPACE:ssl_cipher}) (?:-|%{NOTSPACE:ssl_protocol})
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md
index 381b0ee..cfcf6ed 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -29,10 +29,12 @@ There are two general types types of parsers:
* Grok parser: `org.apache.metron.parsers.GrokParser` with possible `parserConfig` entries of
* `grokPath` : The path in HDFS (or in the Jar) to the grok statement
* `patternLabel` : The pattern label to use from the grok statement
+ * `multiLine` : The raw data passed in should be handled as a long with multiple lines, with each line to be parsed separately. This setting's valid values are 'true' or 'false'. The default if unset is 'false'. When set the parser will handle multiple lines with successfully processed lines emitted normally, and lines with errors sent to the error topic.
* `timestampField` : The field to use for timestamp
* `timeFields` : A list of fields to be treated as time
* `dateFormat` : The date format to use to parse the time fields
* `timezone` : The timezone to use. `UTC` is default.
+ * The Grok parser supports either 1 line to parse per incoming message, or incoming messages with multiple log lines, and will produce a json message per line
* CSV Parser: `org.apache.metron.parsers.csv.CSVParser` with possible `parserConfig` entries of
* `timestampFormat` : The date format of the timestamp to use. If unspecified, the parser assumes the timestamp is ms since unix epoch.
* `columns` : A map of column names you wish to extract from the CSV to their offsets (e.g. `{ 'name' : 1, 'profession' : 3}` would be a column map for extracting the 2nd and 4th columns from a CSV)
@@ -513,9 +515,13 @@ Java parser adapters are intended for higher-velocity topologies and are not eas
* org.apache.metron.parsers.syslog.Syslog5424Parser : Parse Syslog RFC 5424 messages
### Grok Parser Adapters
-Grok parser adapters are designed primarly for someone who is not a Java coder for quickly standing up a parser adapter for lower velocity topologies. Grok relies on Regex for message parsing, which is much slower than purpose-built Java parsers, but is more extensible. Grok parsers are defined via a config file and the topplogy does not need to be recombiled in order to make changes to them. An example of a Grok perser is:
+Grok parser adapters are designed primarily for someone who is not a Java coder for quickly standing up a parser adapter for lower velocity topologies. Grok relies on Regex for message parsing, which is much slower than purpose-built Java parsers, but is more extensible. Grok parsers are defined via a config file and the topplogy does not need to be recompiled in order to make changes to them. Example of a Grok parsers are:
-* org.apache.metron.parsers.GrokParser
+* org.apache.metron.parsers.GrokParser and org.apache.metron.parsers.websphere.GrokWebSphereParser
+
+Parsers that derive from GrokParser typically allow the GrokParser to parse the messages, and then override the methods for postParse to do further parsing.
+When this is the case, and the Parser has not overridden `parse(byte[])` or `parseResultOptional(byte[])` these parsers will gain support for treating byte[] input as multiple lines, with each line parsed as a separate message ( and returned as such).
+This is enabled by using the `"multiline":"true"` Parser configuration option.
For more information on the Grok project please refer to the following link:
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/message-parser-implementation-notes.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/message-parser-implementation-notes.md b/metron-platform/metron-parsers/message-parser-implementation-notes.md
new file mode 100644
index 0000000..b8afe04
--- /dev/null
+++ b/metron-platform/metron-parsers/message-parser-implementation-notes.md
@@ -0,0 +1,57 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# `MessageParser` implementation notes
+
+
+1. Supporting multiple JSONObject returns from a single byte[]
+The original `MessageParser` interface supported parsing a message and returning a `List<JSONObject>`. Therefore explicitly supporting multiple messages from one input.
+While this is fine, it only allows for the complete failure of a message for any reason. There can only be one exception thrown. This means that if there _are_ multiple messages in the buffer, any one failure will necessarily fail all of them.
+To improve on this situation, a new method was added to the `MessageParser` interface (with a default implementation), that introduces a return type to provide not only the JSONObjects produced, but also a `Map` of messages -> throwable.
+
+To support this in your parser, you should:
+
+- Implement the new method
+
+```java
+ @Override
+ public Optional<MessageParserResult<JSONObject>> parseOptionalResult(byte[] rawMessage)
+```
+
+- Implement the original `List<JSONObject> parse(byte[] message)` to delegate to that method such as below:
+
+```java
+ @Override
+ public List<JSONObject> parse(byte[] rawMessage) {
+ Optional<MessageParserResult<JSONObject>> resultOptional = parseOptionalResult(rawMessage);
+ if (!resultOptional.isPresent()) {
+ return Collections.EMPTY_LIST;
+ }
+ Map<Object,Throwable> errors = resultOptional.get().getMessageThrowables();
+ if (!errors.isEmpty()) {
+ throw new RuntimeException(errors.entrySet().iterator().next().getValue());
+ }
+
+ return resultOptional.get().getMessages();
+ }
+```
+
+- You *may* want to govern treating the incoming buffer as multiline or not by adding a configuration option for your parser, such as `"multiline":"true"|"false"`
+
+- See the org.apache.metron.parsers.GrokParser for an example of this implementation.
+
+The Metron system itself will call the new `parseOptionalResult` method during processing. The default implementation in the interface handles backwards compatability with previous implementations.
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/DefaultMessageParserResult.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/DefaultMessageParserResult.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/DefaultMessageParserResult.java
new file mode 100644
index 0000000..11d15eb
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/DefaultMessageParserResult.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers;
+
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class DefaultMessageParserResult<T> implements MessageParserResult<T> {
+ private List<T> messages = new ArrayList<>();
+ private Map<Object, Throwable> errors = new HashMap<>();
+ private Throwable masterThrowable;
+
+ public DefaultMessageParserResult() {
+ }
+
+ public DefaultMessageParserResult(Throwable masterThrowable) {
+ this.masterThrowable = masterThrowable;
+ }
+
+ public DefaultMessageParserResult(List<T> list) {
+ messages.addAll(list);
+ }
+
+ public DefaultMessageParserResult(Map<Object, Throwable> map) {
+ errors.putAll(map);
+ }
+
+ public DefaultMessageParserResult(List<T> list, Map<Object, Throwable> map) {
+ messages.addAll(list);
+ errors.putAll(map);
+ }
+
+ public void addMessage(T message) {
+ messages.add(message);
+ }
+
+ public void addError(Object message, Throwable throwable) {
+ errors.put(message, throwable);
+ }
+
+ @Override
+ public List<T> getMessages() {
+ return messages;
+ }
+
+ @Override
+ public Map<Object, Throwable> getMessageThrowables() {
+ return errors;
+ }
+
+ @Override
+ public Optional<Throwable> getMasterThrowable() {
+ return Optional.ofNullable(masterThrowable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
index 99ac390..bead477 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,32 +15,43 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.metron.parsers;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.common.Constants;
+import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Serializable;
+import java.io.StringReader;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.TimeZone;
-import oi.thekraken.grok.api.Grok;
-import oi.thekraken.grok.api.Match;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.metron.common.Constants;
-import org.apache.metron.parsers.interfaces.MessageParser;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
public class GrokParser implements MessageParser<JSONObject>, Serializable {
@@ -48,6 +59,7 @@ public class GrokParser implements MessageParser<JSONObject>, Serializable {
protected transient Grok grok;
protected String grokPath;
+ protected boolean multiLine = false;
protected String patternLabel;
protected List<String> timeFields = new ArrayList<>();
protected String timestampField;
@@ -55,8 +67,13 @@ public class GrokParser implements MessageParser<JSONObject>, Serializable {
protected String patternsCommonDir = "/patterns/common";
@Override
+ @SuppressWarnings("unchecked")
public void configure(Map<String, Object> parserConfig) {
this.grokPath = (String) parserConfig.get("grokPath");
+ String multiLineString = (String) parserConfig.get("multiLine");
+ if (!StringUtils.isBlank(multiLineString)) {
+ multiLine = Boolean.parseBoolean(multiLineString);
+ }
this.patternLabel = (String) parserConfig.get("patternLabel");
this.timestampField = (String) parserConfig.get("timestampField");
List<String> timeFieldsParam = (List<String>) parserConfig.get("timeFields");
@@ -127,10 +144,87 @@ public class GrokParser implements MessageParser<JSONObject>, Serializable {
@SuppressWarnings("unchecked")
@Override
public List<JSONObject> parse(byte[] rawMessage) {
+ Optional<MessageParserResult<JSONObject>> resultOptional = parseOptionalResult(rawMessage);
+ if (!resultOptional.isPresent()) {
+ return Collections.EMPTY_LIST;
+ }
+ Map<Object,Throwable> errors = resultOptional.get().getMessageThrowables();
+ if (!errors.isEmpty()) {
+ throw new RuntimeException(errors.entrySet().iterator().next().getValue());
+ }
+
+ return resultOptional.get().getMessages();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Optional<MessageParserResult<JSONObject>> parseOptionalResult(byte[] rawMessage) {
if (grok == null) {
init();
}
+ if (multiLine) {
+ return parseMultiLine(rawMessage);
+ }
+ return parseSingleLine(rawMessage);
+ }
+
+ @SuppressWarnings("unchecked")
+ private Optional<MessageParserResult<JSONObject>> parseMultiLine(byte[] rawMessage) {
+ List<JSONObject> messages = new ArrayList<>();
+ Map<Object,Throwable> errors = new HashMap<>();
+ String originalMessage = null;
+ // read the incoming raw data as if it may have multiple lines of logs
+ // if there is only only one line, it will just get processed.
+ try (BufferedReader reader = new BufferedReader(new StringReader(new String(rawMessage, StandardCharsets.UTF_8)))) {
+ while ((originalMessage = reader.readLine()) != null) {
+ LOG.debug("Grok parser parsing message: {}", originalMessage);
+ try {
+ Match gm = grok.match(originalMessage);
+ gm.captures();
+ JSONObject message = new JSONObject();
+ message.putAll(gm.toMap());
+
+ if (message.size() == 0) {
+ Throwable rte = new RuntimeException("Grok statement produced a null message. Original message was: "
+ + originalMessage + " and the parsed message was: " + message + " . Check the pattern at: "
+ + grokPath);
+ errors.put(originalMessage, rte);
+ continue;
+ }
+ message.put("original_string", originalMessage);
+ for (String timeField : timeFields) {
+ String fieldValue = (String) message.get(timeField);
+ if (fieldValue != null) {
+ message.put(timeField, toEpoch(fieldValue));
+ }
+ }
+ if (timestampField != null) {
+ message.put(Constants.Fields.TIMESTAMP.getName(), formatTimestamp(message.get(timestampField)));
+ }
+ message.remove(patternLabel);
+ postParse(message);
+ messages.add(message);
+ LOG.debug("Grok parser parsed message: {}", message);
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ errors.put(originalMessage, e);
+ }
+ }
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ Exception innerException = new IllegalStateException("Grok parser Error: "
+ + e.getMessage()
+ + " on "
+ + originalMessage, e);
+ return Optional.of(new DefaultMessageParserResult<>(innerException));
+ }
+ return Optional.of(new DefaultMessageParserResult<>(messages, errors));
+ }
+
+ @SuppressWarnings("unchecked")
+ private Optional<MessageParserResult<JSONObject>> parseSingleLine(byte[] rawMessage) {
List<JSONObject> messages = new ArrayList<>();
+ Map<Object,Throwable> errors = new HashMap<>();
String originalMessage = null;
try {
originalMessage = new String(rawMessage, "UTF-8");
@@ -140,30 +234,36 @@ public class GrokParser implements MessageParser<JSONObject>, Serializable {
JSONObject message = new JSONObject();
message.putAll(gm.toMap());
- if (message.size() == 0)
- throw new RuntimeException("Grok statement produced a null message. Original message was: "
+ if (message.size() == 0) {
+ Throwable rte = new RuntimeException("Grok statement produced a null message. Original message was: "
+ originalMessage + " and the parsed message was: " + message + " . Check the pattern at: "
+ grokPath);
-
- message.put("original_string", originalMessage);
- for (String timeField : timeFields) {
- String fieldValue = (String) message.get(timeField);
- if (fieldValue != null) {
- message.put(timeField, toEpoch(fieldValue));
+ errors.put(originalMessage, rte);
+ } else {
+ message.put("original_string", originalMessage);
+ for (String timeField : timeFields) {
+ String fieldValue = (String) message.get(timeField);
+ if (fieldValue != null) {
+ message.put(timeField, toEpoch(fieldValue));
+ }
}
+ if (timestampField != null) {
+ message.put(Constants.Fields.TIMESTAMP.getName(), formatTimestamp(message.get(timestampField)));
+ }
+ message.remove(patternLabel);
+ postParse(message);
+ messages.add(message);
+ LOG.debug("Grok parser parsed message: {}", message);
}
- if (timestampField != null) {
- message.put(Constants.Fields.TIMESTAMP.getName(), formatTimestamp(message.get(timestampField)));
- }
- message.remove(patternLabel);
- postParse(message);
- messages.add(message);
- LOG.debug("Grok parser parsed message: {}", message);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
- throw new IllegalStateException("Grok parser Error: " + e.getMessage() + " on " + originalMessage , e);
+ Exception innerException = new IllegalStateException("Grok parser Error: "
+ + e.getMessage()
+ + " on "
+ + originalMessage, e);
+ return Optional.of(new DefaultMessageParserResult<>(innerException));
}
- return messages;
+ return Optional.of(new DefaultMessageParserResult<JSONObject>(messages, errors));
}
@Override
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
index 213d02c..ff5c1d4 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,23 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.metron.parsers.bolt;
import com.github.benmanes.caffeine.cache.Cache;
-import java.io.Serializable;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Optional;
-import java.util.Set;
-import java.util.UUID;
-import java.util.function.Function;
-import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.metron.common.Constants;
import org.apache.metron.common.bolt.ConfiguredParserBolt;
@@ -48,6 +36,7 @@ import org.apache.metron.common.utils.ErrorUtils;
import org.apache.metron.parsers.filters.Filters;
import org.apache.metron.parsers.interfaces.MessageFilter;
import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
import org.apache.metron.parsers.topology.ParserComponents;
import org.apache.metron.stellar.common.CachingStellarProcessor;
import org.apache.metron.stellar.dsl.Context;
@@ -66,6 +55,20 @@ import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.Serializable;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
public class ParserBolt extends ConfiguredParserBolt implements Serializable {
@@ -306,8 +309,17 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable {
);
metadata = rawMessage.getMetadata();
- Optional<List<JSONObject>> messages = parser.parseOptional(rawMessage.getMessage());
- for (JSONObject message : messages.orElse(Collections.emptyList())) {
+ Optional<MessageParserResult<JSONObject>> results = parser.parseOptionalResult(rawMessage.getMessage());
+
+ // check if there is a master error
+ if (results.isPresent() && results.get().getMasterThrowable().isPresent()) {
+ handleError(originalMessage, tuple, results.get().getMasterThrowable().get(), collector);
+ return;
+ }
+
+ // Handle the message results
+ List<JSONObject> messages = results.isPresent() ? results.get().getMessages() : Collections.EMPTY_LIST;
+ for (JSONObject message : messages) {
//we want to ack the tuple in the situation where we have are not doing a bulk write
//otherwise we want to defer to the writerComponent who will ack on bulk commit.
WriterHandler writer = sensorToComponentMap.get(sensor).getWriter();
@@ -370,6 +382,19 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable {
}
}
}
+
+ // Handle the error results
+ Map<Object, Throwable> messageErrors = results.isPresent()
+ ? results.get().getMessageThrowables() : Collections.EMPTY_MAP;
+
+ for (Entry<Object,Throwable> entry : messageErrors.entrySet()) {
+ MetronError error = new MetronError()
+ .withErrorType(Constants.ErrorType.PARSER_ERROR)
+ .withThrowable(entry.getValue())
+ .withSensorType(sensorToComponentMap.keySet())
+ .addRawMessage(originalMessage);
+ ErrorUtils.handleError(collector, error);
+ }
}
//if we are supposed to ack the tuple OR if we've never passed this tuple to the bulk writer
//(meaning that none of the messages are valid either globally or locally)
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
index e3b903e..f8243b9 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
@@ -17,7 +17,11 @@
*/
package org.apache.metron.parsers.interfaces;
+import org.apache.metron.parsers.DefaultMessageParserResult;
+
import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -31,14 +35,14 @@ public interface MessageParser<T> extends Configurable {
/**
* Take raw data and convert it to a list of messages.
*
- * @param rawMessage
+ * @param rawMessage the raw bytes of the message
* @return If null is returned, this is treated as an empty list.
*/
List<T> parse(byte[] rawMessage);
/**
* Take raw data and convert it to an optional list of messages.
- * @param parseMessage
+ * @param parseMessage the raw bytes of the message
* @return If null is returned, this is treated as an empty list.
*/
default Optional<List<T>> parseOptional(byte[] parseMessage) {
@@ -46,8 +50,26 @@ public interface MessageParser<T> extends Configurable {
}
/**
+ * Take raw data and convert it to messages. Each raw message may produce multiple messages and therefore
+ * multiple errors. A {@link MessageParserResult} is returned, which will have both the messages produced
+ * and the errors.
+ * @param parseMessage the raw bytes of the message
+ * @return Optional of {@link MessageParserResult}
+ */
+ default Optional<MessageParserResult<T>> parseOptionalResult(byte[] parseMessage) {
+ List<T> list = new ArrayList<>();
+ try {
+ Optional<List<T>> optionalMessages = parseOptional(parseMessage);
+ optionalMessages.ifPresent(list::addAll);
+ } catch (Throwable t) {
+ return Optional.of(new DefaultMessageParserResult<>(t));
+ }
+ return Optional.of(new DefaultMessageParserResult<T>(list));
+ }
+
+ /**
* Validate the message to ensure that it's correct.
- * @param message
+ * @param message the message to validate
* @return true if the message is valid, false if not
*/
boolean validate(T message);
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParserResult.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParserResult.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParserResult.java
new file mode 100644
index 0000000..891e94f
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParserResult.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.interfaces;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Result object MessageParser calls.
+ * @param <T>
+ */
+public interface MessageParserResult<T> {
+ /**
+ * Returns the Message objects of {@code T}
+ * @return {@code List}
+ */
+ List<T> getMessages();
+
+ /**
+ * Returns a map of raw message objects to the {@code Throwable} they triggered.
+ * @return {@code Map}
+ */
+ Map<Object,Throwable> getMessageThrowables();
+
+ /**
+ * Returns a master {@code Throwable} for a parse call. This represents a complete
+ * call failure, as opposed to one associated with a message.
+ * @return {@code Optional}{@code Throwable}
+ */
+ Optional<Throwable> getMasterThrowable();
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java
index 178719b..a58e0c9 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/websphere/GrokWebSphereParser.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -27,117 +27,114 @@ import java.util.Iterator;
public class GrokWebSphereParser extends GrokParser {
- private static final long serialVersionUID = 4860439408055777358L;
+ private static final long serialVersionUID = 4860439408055777358L;
- @Override
- protected long formatTimestamp(Object value) {
- long epochTimestamp = System.currentTimeMillis();
- if (value != null) {
- try {
- epochTimestamp = toEpoch(Calendar.getInstance().get(Calendar.YEAR) + " " + value);
- } catch (ParseException e) {
- //default to current time
- }
- }
- return epochTimestamp;
- }
+ @Override
+ protected long formatTimestamp(Object value) {
+ long epochTimestamp = System.currentTimeMillis();
+ if (value != null) {
+ try {
+ epochTimestamp = toEpoch(Calendar.getInstance().get(Calendar.YEAR) + " " + value);
+ } catch (ParseException e) {
+ //default to current time
+ }
+ }
+ return epochTimestamp;
+ }
- @Override
- protected void postParse(JSONObject message) {
- removeEmptyFields(message);
- message.remove("timestamp_string");
- if (message.containsKey("message")) {
- String messageValue = (String) message.get("message");
- if (messageValue.contains("logged into")) {
- parseLoginMessage(message);
- }
- else if (messageValue.contains("logged out")) {
- parseLogoutMessage(message);
- }
- else if (messageValue.contains("rbm(")) {
- parseRBMMessage(message);
- }
- else {
- parseOtherMessage(message);
- }
- }
- }
+ @Override
+ protected void postParse(JSONObject message) {
+ removeEmptyFields(message);
+ message.remove("timestamp_string");
+ if (message.containsKey("message")) {
+ String messageValue = (String) message.get("message");
+ if (messageValue.contains("logged into")) {
+ parseLoginMessage(message);
+ } else if (messageValue.contains("logged out")) {
+ parseLogoutMessage(message);
+ } else if (messageValue.contains("rbm(")) {
+ parseRBMMessage(message);
+ } else {
+ parseOtherMessage(message);
+ }
+ }
+ }
- @SuppressWarnings("unchecked")
- private void removeEmptyFields(JSONObject json) {
- Iterator<Object> keyIter = json.keySet().iterator();
- while (keyIter.hasNext()) {
- Object key = keyIter.next();
- Object value = json.get(key);
- if (null == value || "".equals(value.toString())) {
- keyIter.remove();
- }
- }
- }
+ @SuppressWarnings("unchecked")
+ private void removeEmptyFields(JSONObject json) {
+ Iterator<Object> keyIter = json.keySet().iterator();
+ while (keyIter.hasNext()) {
+ Object key = keyIter.next();
+ Object value = json.get(key);
+ if (null == value || "".equals(value.toString())) {
+ keyIter.remove();
+ }
+ }
+ }
- //Extracts the appropriate fields from login messages
- @SuppressWarnings("unchecked")
- private void parseLoginMessage(JSONObject json) {
- json.put("event_subtype", "login");
- String message = (String) json.get("message");
- if (message.contains(":")){
- String parts[] = message.split(":");
- String user = parts[0];
- String ip_src_addr = parts[1];
- if (user.contains("user(") && user.contains(")")) {
- user = user.substring(user.indexOf("user(") + "user(".length());
- user = user.substring(0, user.indexOf(")"));
- json.put("username", user);
- }
- if (ip_src_addr.contains("[") && ip_src_addr.contains("]")) {
- ip_src_addr = ip_src_addr.substring(ip_src_addr.indexOf("[") + 1);
- ip_src_addr = ip_src_addr.substring(0, ip_src_addr.indexOf("]"));
- json.put("ip_src_addr", ip_src_addr);
- }
- json.remove("message");
- }
- }
+ //Extracts the appropriate fields from login messages
+ @SuppressWarnings("unchecked")
+ private void parseLoginMessage(JSONObject json) {
+ json.put("event_subtype", "login");
+ String message = (String) json.get("message");
+ if (message.contains(":")) {
+ String[] parts = message.split(":");
+ String user = parts[0];
+ String ip_src_addr = parts[1];
+ if (user.contains("user(") && user.contains(")")) {
+ user = user.substring(user.indexOf("user(") + "user(".length());
+ user = user.substring(0, user.indexOf(")"));
+ json.put("username", user);
+ }
+ if (ip_src_addr.contains("[") && ip_src_addr.contains("]")) {
+ ip_src_addr = ip_src_addr.substring(ip_src_addr.indexOf("[") + 1);
+ ip_src_addr = ip_src_addr.substring(0, ip_src_addr.indexOf("]"));
+ json.put("ip_src_addr", ip_src_addr);
+ }
+ json.remove("message");
+ }
+ }
- //Extracts the appropriate fields from logout messages
- @SuppressWarnings("unchecked")
- private void parseLogoutMessage(JSONObject json) {
- json.put("event_subtype", "logout");
- String message = (String) json.get("message");
- if (message.matches(".*'.*'.*'.*'.*")) {
- String parts[] = message.split("'");
- String ip_src_addr = parts[0];
- if (ip_src_addr.contains("[") && ip_src_addr.contains("]")) {
- ip_src_addr = ip_src_addr.substring(ip_src_addr.indexOf("[") + 1);
- ip_src_addr = ip_src_addr.substring(0, ip_src_addr.indexOf("]"));
- json.put("ip_src_addr", ip_src_addr);
- }
- json.put("username", parts[1]);
- json.put("security_domain", parts[3]);
- json.remove("message");
- }
- }
+ //Extracts the appropriate fields from logout messages
+ @SuppressWarnings("unchecked")
+ private void parseLogoutMessage(JSONObject json) {
+ json.put("event_subtype", "logout");
+ String message = (String) json.get("message");
+ if (message.matches(".*'.*'.*'.*'.*")) {
+ String parts[] = message.split("'");
+ String ip_src_addr = parts[0];
+ if (ip_src_addr.contains("[") && ip_src_addr.contains("]")) {
+ ip_src_addr = ip_src_addr.substring(ip_src_addr.indexOf("[") + 1);
+ ip_src_addr = ip_src_addr.substring(0, ip_src_addr.indexOf("]"));
+ json.put("ip_src_addr", ip_src_addr);
+ }
+ json.put("username", parts[1]);
+ json.put("security_domain", parts[3]);
+ json.remove("message");
+ }
+ }
- //Extracts the appropriate fields from RBM messages
- @SuppressWarnings("unchecked")
- private void parseRBMMessage(JSONObject json) {
- String message = (String) json.get("message");
- if (message.contains("(")) {
- json.put("process", message.substring(0, message.indexOf("(")));
- if (message.contains(":")) {
- json.put("message", message.substring(message.indexOf(":") + 2));
- }
- }
- }
+ //Extracts the appropriate fields from RBM messages
+ @SuppressWarnings("unchecked")
+ private void parseRBMMessage(JSONObject json) {
+ String message = (String) json.get("message");
+ if (message.contains("(")) {
+ json.put("process", message.substring(0, message.indexOf("(")));
+ if (message.contains(":")) {
+ json.put("message", message.substring(message.indexOf(":") + 2));
+ }
+ }
+ }
- //Extracts the appropriate fields from other messages
- @SuppressWarnings("unchecked")
- private void parseOtherMessage(JSONObject json) {
- String message = (String) json.get("message");
- if (message.contains("(")) {
- json.put("process", message.substring(0, message.indexOf("(")));
- if (message.contains(":")) {
- json.put("message", message.substring(message.indexOf(":") + 2));
- }
- }
- }
+ //Extracts the appropriate fields from other messages
+ @SuppressWarnings("unchecked")
+ private void parseOtherMessage(JSONObject json) {
+ String message = (String) json.get("message");
+ if (message.contains("(")) {
+ json.put("process", message.substring(0, message.indexOf("(")));
+ if (message.contains(":")) {
+ json.put("message", message.substring(message.indexOf(":") + 2));
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java
index 1a50dea..7fa6a31 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/GrokParserTest.java
@@ -92,4 +92,5 @@ public abstract class GrokParserTest {
public abstract List<String> getTimeFields();
public abstract String getDateFormat();
public abstract String getTimestampField();
+ public abstract String getMultiLine();
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
new file mode 100644
index 0000000..cc4d20f
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class MultiLineGrokParserTest {
+
+ /**
+ * Test that if a byte[] with multiple lines of log is passed in
+ * it will be parsed into the correct number of messages.
+ * @throws IOException if we can't read from disk
+ * @throws ParseException if we can't parse
+ */
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testLegacyInterfaceReturnsMultiline() throws IOException, ParseException {
+
+ Map<String, Object> parserConfig = new HashMap<>();
+ parserConfig.put("grokPath", getGrokPath());
+ parserConfig.put("patternLabel", getGrokPatternLabel());
+ parserConfig.put("timestampField", getTimestampField());
+ parserConfig.put("dateFormat", getDateFormat());
+ parserConfig.put("timeFields", getTimeFields());
+ parserConfig.put("multiLine", getMultiLine());
+ GrokParser grokParser = new GrokParser();
+ grokParser.configure(parserConfig);
+ grokParser.init();
+
+ JSONParser jsonParser = new JSONParser();
+ Map<String, String> testData = getTestData();
+ for (Map.Entry<String, String> e : testData.entrySet()) {
+ byte[] rawMessage = e.getKey().getBytes();
+ List<JSONObject> parsedList = grokParser.parse(rawMessage);
+ Assert.assertEquals(10, parsedList.size());
+ }
+ }
+
+ /**
+ * Test that if a byte[] with multiple lines of log is passed in
+ * it will be parsed into the correct number of messages using the
+ * parseOptionalResult call.
+ * @throws IOException if we can't read from disk
+ * @throws ParseException if we can't parse
+ */
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testOptionalResultReturnsMultiline() throws IOException, ParseException {
+
+ Map<String, Object> parserConfig = new HashMap<>();
+ parserConfig.put("grokPath", getGrokPath());
+ parserConfig.put("patternLabel", getGrokPatternLabel());
+ parserConfig.put("timestampField", getTimestampField());
+ parserConfig.put("dateFormat", getDateFormat());
+ parserConfig.put("timeFields", getTimeFields());
+ parserConfig.put("multiLine", getMultiLine());
+
+ GrokParser grokParser = new GrokParser();
+ grokParser.configure(parserConfig);
+ grokParser.init();
+
+ JSONParser jsonParser = new JSONParser();
+ Map<String, String> testData = getTestData();
+ for (Map.Entry<String, String> e : testData.entrySet()) {
+ byte[] rawMessage = e.getKey().getBytes();
+ Optional<MessageParserResult<JSONObject>> resultOptional = grokParser.parseOptionalResult(rawMessage);
+ Assert.assertTrue(resultOptional.isPresent());
+ Optional<Throwable> throwableOptional = resultOptional.get().getMasterThrowable();
+ List<JSONObject> resultList = resultOptional.get().getMessages();
+ Map<Object,Throwable> errorMap = resultOptional.get().getMessageThrowables();
+ Assert.assertFalse(throwableOptional.isPresent());
+ Assert.assertEquals(0, errorMap.size());
+ Assert.assertEquals(10, resultList.size());
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map getTestData() {
+
+ Map testData = new HashMap<String, String>();
+ String input;
+ try (FileInputStream stream = new FileInputStream(new File("src/test/resources/logData/multi_elb_log.txt"))) {
+ input = IOUtils.toString(stream);
+ } catch (IOException ioe) {
+ throw new IllegalStateException("failed to open file", ioe);
+ }
+ // not checking values, just that we get the right number of messages
+ testData.put(input, "");
+ return testData;
+
+ }
+
+ public String getMultiLine() { return "true";}
+
+ public String getGrokPath() {
+ return "../metron-integration-test/src/main/sample/patterns/test";
+ }
+
+ public String getGrokPatternLabel() {
+ return "ELBACCESSLOGS";
+ }
+
+ public List<String> getTimeFields() {
+ return new ArrayList<String>() {{
+ add("timestamp");
+ }};
+ }
+
+ public String getDateFormat() {
+ return "yyyy-MM-dd'T'HH:mm:ss.S'Z'";
+ }
+
+ public String getTimestampField() {
+ return "timestamp";
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java
new file mode 100644
index 0000000..8ab8246
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineWithErrorsGrokParserTest.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class MultiLineWithErrorsGrokParserTest {
+
+ /**
+ * Test that if a byte[] with multiple lines of log is passed in
+ * it will be parsed into the correct number of messages.
+ * @throws IOException if we can't read from disk
+ * @throws ParseException if we can't parse
+ */
+ @Test(expected = RuntimeException.class)
+ @SuppressWarnings("unchecked")
+ public void testLegacyInterfaceThrowsOneExceptionWithMultiline() throws IOException, ParseException {
+
+ Map<String, Object> parserConfig = new HashMap<>();
+ parserConfig.put("grokPath", getGrokPath());
+ parserConfig.put("patternLabel", getGrokPatternLabel());
+ parserConfig.put("timestampField", getTimestampField());
+ parserConfig.put("dateFormat", getDateFormat());
+ parserConfig.put("timeFields", getTimeFields());
+ parserConfig.put("multiLine",getMultiLine());
+
+ GrokParser grokParser = new GrokParser();
+ grokParser.configure(parserConfig);
+ grokParser.init();
+
+ JSONParser jsonParser = new JSONParser();
+ Map<String, String> testData = getTestData();
+ for (Map.Entry<String, String> e : testData.entrySet()) {
+ byte[] rawMessage = e.getKey().getBytes();
+ List<JSONObject> parsedList = grokParser.parse(rawMessage);
+ }
+ }
+
+ /**
+ * Test that if a byte[] with multiple lines of log is passed in
+ * it will be parsed into the correct number of messages using the
+ * parseOptionalResult call.
+ * @throws IOException if we can't read from disk
+ * @throws ParseException if we can't parse
+ */
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testResultInterfaceReturnsErrorsAndMessagesWithMultiline() throws IOException, ParseException {
+
+ Map<String, Object> parserConfig = new HashMap<>();
+ parserConfig.put("grokPath", getGrokPath());
+ parserConfig.put("patternLabel", getGrokPatternLabel());
+ parserConfig.put("timestampField", getTimestampField());
+ parserConfig.put("dateFormat", getDateFormat());
+ parserConfig.put("timeFields", getTimeFields());
+ parserConfig.put("multiLine",getMultiLine());
+
+ GrokParser grokParser = new GrokParser();
+ grokParser.configure(parserConfig);
+ grokParser.init();
+
+ JSONParser jsonParser = new JSONParser();
+ Map<String, String> testData = getTestData();
+ for (Map.Entry<String, String> e : testData.entrySet()) {
+ byte[] rawMessage = e.getKey().getBytes();
+ Optional<MessageParserResult<JSONObject>> resultOptional = grokParser.parseOptionalResult(rawMessage);
+ Assert.assertTrue(resultOptional.isPresent());
+ Optional<Throwable> throwableOptional = resultOptional.get().getMasterThrowable();
+ List<JSONObject> resultList = resultOptional.get().getMessages();
+ Map<Object,Throwable> errorMap = resultOptional.get().getMessageThrowables();
+ Assert.assertFalse(throwableOptional.isPresent());
+ Assert.assertEquals(3, errorMap.size());
+ Assert.assertEquals(10, resultList.size());
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map getTestData() {
+
+ Map testData = new HashMap<String, String>();
+ String input;
+ try (FileInputStream stream = new FileInputStream(new File("src/test/resources/logData/multi_elb_with_errors_log.txt"))) {
+ input = IOUtils.toString(stream);
+ } catch (IOException ioe) {
+ throw new IllegalStateException("failed to open file", ioe);
+ }
+ // not checking values, just that we get the right number of messages
+ testData.put(input, "");
+ return testData;
+
+ }
+
+ public String getGrokPath() {
+ return "../metron-integration-test/src/main/sample/patterns/test";
+ }
+
+ public String getGrokPatternLabel() {
+ return "ELBACCESSLOGS";
+ }
+
+ public List<String> getTimeFields() {
+ return new ArrayList<String>() {{
+ add("timestamp");
+ }};
+ }
+
+ public String getMultiLine() { return "true"; }
+
+ public String getDateFormat() {
+ return "yyyy-MM-dd'T'HH:mm:ss.S'Z'";
+ }
+
+ public String getTimestampField() {
+ return "timestamp";
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java
index e060559..35e07f8 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SampleGrokParserTest.java
@@ -69,6 +69,9 @@ public class SampleGrokParserTest extends GrokParserTest {
}
@Override
+ public String getMultiLine() { return "false"; }
+
+ @Override
public String getGrokPath() {
return "../metron-integration-test/src/main/sample/patterns/test";
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java
index 93c8276..cb424fb 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SquidParserTest.java
@@ -73,6 +73,8 @@ public class SquidParserTest extends GrokParserTest {
}
+ @Override
+ public String getMultiLine() { return "false"; }
@Override
public String getGrokPath() {
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java
index 8dd75a0..15ce19f 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/YafParserTest.java
@@ -69,6 +69,9 @@ public class YafParserTest extends GrokParserTest {
}
@Override
+ public String getMultiLine() { return "false"; }
+
+ @Override
public String getGrokPath() {
return "../metron-parsers/src/main/resources/patterns/yaf";
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
index 06f4cec..0ae2817 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
@@ -56,6 +56,7 @@ import org.apache.metron.common.writer.BulkWriterResponse;
import org.apache.metron.common.writer.MessageWriter;
import org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater;
import org.apache.metron.parsers.BasicParser;
+import org.apache.metron.parsers.DefaultMessageParserResult;
import org.apache.metron.parsers.interfaces.MessageFilter;
import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.parsers.topology.ParserComponents;
@@ -254,7 +255,7 @@ public class ParserBoltTest extends BaseBoltTest {
parsedMessage.put("guid", "this-is-unique-identifier-for-tuple");
List<JSONObject> messageList = new ArrayList<>();
messageList.add(parsedMessage);
- when(parser.parseOptional(sampleBinary)).thenReturn(Optional.of(messageList));
+ when(parser.parseOptionalResult(sampleBinary)).thenReturn(Optional.of(new DefaultMessageParserResult<>(messageList)));
when(parser.validate(parsedMessage)).thenReturn(true);
parserBolt.execute(tuple);
@@ -303,7 +304,7 @@ public class ParserBoltTest extends BaseBoltTest {
final JSONObject finalMessage1 = (JSONObject) jsonParser.parse("{ \"field1\":\"value1\", \"source.type\":\"" + sensorType + "\", \"guid\": \"this-is-unique-identifier-for-tuple\" }");
final JSONObject finalMessage2 = (JSONObject) jsonParser.parse("{ \"field2\":\"value2\", \"source.type\":\"" + sensorType + "\", \"guid\": \"this-is-unique-identifier-for-tuple\" }");
when(tuple.getBinary(0)).thenReturn(sampleBinary);
- when(parser.parseOptional(sampleBinary)).thenReturn(Optional.of(messages));
+ when(parser.parseOptionalResult(sampleBinary)).thenReturn(Optional.of(new DefaultMessageParserResult<>(messages)));
when(parser.validate(eq(messages.get(0)))).thenReturn(true);
when(parser.validate(eq(messages.get(1)))).thenReturn(false);
parserBolt.execute(tuple);
@@ -358,9 +359,9 @@ public class ParserBoltTest extends BaseBoltTest {
when(successResponse.getSuccesses()).thenReturn(ImmutableList.of(t1));
when(batchWriter.write(any(), any(), any(), any())).thenReturn(successResponse);
when(parser.validate(any())).thenReturn(true);
- when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject(new HashMap<String, Object>() {{
+ when(parser.parseOptionalResult(any())).thenReturn(Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject(new HashMap<String, Object>() {{
put("field1", "blah");
- }}))));
+ }})))));
parserBolt.execute(t1);
verify(batchWriter, times(1)).write(any(), any(), any(), any());
verify(outputCollector, times(1)).ack(t1);
@@ -543,7 +544,7 @@ public class ParserBoltTest extends BaseBoltTest {
verify(parser, times(1)).init();
verify(batchWriter, times(1)).init(any(), any(), any());
when(parser.validate(any())).thenReturn(true);
- when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject())));
+ when(parser.parseOptionalResult(any())).thenReturn(Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject()))));
when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
int oneLessThanDefaultBatchSize = ParserConfigurations.DEFAULT_KAFKA_BATCH_SIZE - 1;
BulkWriterResponse response = new BulkWriterResponse();
@@ -624,7 +625,7 @@ public class ParserBoltTest extends BaseBoltTest {
verify(parser, times(1)).init();
verify(batchWriter, times(1)).init(any(), any(), any());
when(parser.validate(any())).thenReturn(true);
- when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject())));
+ when(parser.parseOptionalResult(any())).thenReturn(Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject()))));
when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
Set<Tuple> tuples = Stream.of(t1, t2, t3, t4, t5).collect(Collectors.toSet());
BulkWriterResponse response = new BulkWriterResponse();
@@ -671,7 +672,7 @@ public class ParserBoltTest extends BaseBoltTest {
doThrow(new Exception()).when(batchWriter).write(any(), any(), any(), any());
when(parser.validate(any())).thenReturn(true);
- when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject())));
+ when(parser.parseOptionalResult(any())).thenReturn(Optional.of(new DefaultMessageParserResult<>(ImmutableList.of(new JSONObject()))));
when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
parserBolt.execute(t1);
parserBolt.execute(t2);
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
index 2cba40a..0d6eef8 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
@@ -130,11 +130,11 @@ public class ParserDriver implements Serializable {
LOG.error("Error parsing message: " + ex.getMessage(), ex);
}
+ @SuppressWarnings("unchecked")
public ProcessorResult<List<byte[]>> getResults() {
return new ProcessorResult.Builder<List<byte[]>>().withProcessErrors(errors)
.withResult(output)
.build();
-
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
index 230c147..2923a4f 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/websphere/GrokWebSphereParserTest.java
@@ -244,17 +244,4 @@ public class GrokWebSphereParserTest {
assertEquals("trans 191) admindefaultsystem*): ntp-service 'NTP Service' - Operational state down:", parsedJSON.get("message"));
}
-
- @Test(expected=RuntimeException.class)
- public void testParseEmptyLine() throws Exception {
-
- //Set up parser, attempt to parse malformed message
- GrokWebSphereParser parser = new GrokWebSphereParser();
- parser.configure(parserConfig);
- String testString = "";
- UnitTestHelper.setLog4jLevel(GrokParser.class, Level.FATAL);
- List<JSONObject> result = parser.parse(testString.getBytes());
- UnitTestHelper.setLog4jLevel(GrokParser.class, Level.ERROR);
- }
-
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/test/resources/logData/multi_elb_log.txt
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/multi_elb_log.txt b/metron-platform/metron-parsers/src/test/resources/logData/multi_elb_log.txt
new file mode 100644
index 0000000..95d3fec
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/resources/logData/multi_elb_log.txt
@@ -0,0 +1,10 @@
+2018-08-30T21:02:32.047266Z vault-development 192.168.113.53:60002 192.168.82.130:8200 0.000695 0.000017 0.000015 - - 607 3078 "- - - " "-" - -
+2018-08-30T21:02:22.119595Z vault-development 192.168.113.29:57322 192.168.72.75:8200 0.00051 0.00001 0.000013 - - 607 3079 "- - - " "-" - -
+2018-08-30T21:05:58.275961Z vault-production 192.168.205.159:58390 192.168.68.196:8200 0.000767 0.000009 0.000012 - - 673 3210 "- - - " "-" - -
+2018-08-30T21:05:59.222277Z vault-production 192.168.228.182:26358 192.168.81.224:8200 0.000882 0.000014 0.000024 - - 519 3920 "- - - " "-" - -
+2018-08-30T21:05:59.234471Z vault-production 192.168.228.182:35506 192.168.79.6:8200 0.000377 0.000011 0.000009 - - 519 3919 "- - - " "-" - -
+2018-08-30T21:05:59.237375Z vault-production 192.168.228.182:52516 192.168.68.196:8200 0.000628 0.000007 0.00001 - - 519 3918 "- - - " "-" - -
+2018-08-30T21:06:05.235460Z vault-production 192.168.228.182:41783 192.168.79.6:8200 0.000309 0.000006 0.00001 - - 519 3918 "- - - " "-" - -
+2018-08-30T21:06:05.226698Z vault-production 192.168.228.182:40008 192.168.81.224:8200 0.000955 0.000014 0.000013 - - 519 3919 "- - - " "-" - -
+2018-08-30T21:06:05.237946Z vault-production 192.168.228.182:19261 192.168.68.196:8200 0.000661 0.000006 0.000009 - - 519 3918 "- - - " "-" - -
+2018-08-30T21:06:11.229542Z vault-production 192.168.228.182:44082 192.168.81.224:8200 0.000912 0.000009 0.000014 - - 519 3919 "- - - " "-" - -
http://git-wip-us.apache.org/repos/asf/metron/blob/3d923cde/metron-platform/metron-parsers/src/test/resources/logData/multi_elb_with_errors_log.txt
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/multi_elb_with_errors_log.txt b/metron-platform/metron-parsers/src/test/resources/logData/multi_elb_with_errors_log.txt
new file mode 100644
index 0000000..3525fc4
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/resources/logData/multi_elb_with_errors_log.txt
@@ -0,0 +1,13 @@
+2018-08-30T21:02:32.047266Z vault-development 192.168.113.53:60002 192.168.82.130:8200 0.000695 0.000017 0.000015 - - 607 3078 "- - - " "-" - -
+2018-08-30T21:02:22.119595Z vault-development 192.168.113.29:57322 192.168.72.75:8200 0.00051 0.00001 0.000013 - - 607 3079 "- - - " "-" - -
+2018-08-30T21:05:58.275961Z vault-production 192.168.205.159:58390 192.168.68.196:8200 0.000767 0.000009 0.000012 - - 673 3210 "- - - " "-" - -
+2018-08-30T21:05:59.222277Z vault-production 192.168.228.182:26358 192.168.81.224:8200 0.000882 0.000014 0.000024 - - 519 3920 "- - - " "-" - -
+BOOM
+BLAM
+BOP
+2018-08-30T21:05:59.234471Z vault-production 192.168.228.182:35506 192.168.79.6:8200 0.000377 0.000011 0.000009 - - 519 3919 "- - - " "-" - -
+2018-08-30T21:05:59.237375Z vault-production 192.168.228.182:52516 192.168.68.196:8200 0.000628 0.000007 0.00001 - - 519 3918 "- - - " "-" - -
+2018-08-30T21:06:05.235460Z vault-production 192.168.228.182:41783 192.168.79.6:8200 0.000309 0.000006 0.00001 - - 519 3918 "- - - " "-" - -
+2018-08-30T21:06:05.226698Z vault-production 192.168.228.182:40008 192.168.81.224:8200 0.000955 0.000014 0.000013 - - 519 3919 "- - - " "-" - -
+2018-08-30T21:06:05.237946Z vault-production 192.168.228.182:19261 192.168.68.196:8200 0.000661 0.000006 0.000009 - - 519 3918 "- - - " "-" - -
+2018-08-30T21:06:11.229542Z vault-production 192.168.228.182:44082 192.168.81.224:8200 0.000912 0.000009 0.000014 - - 519 3919 "- - - " "-" - -
[13/51] [abbrv] metron git commit: METRON-1508 In Ubuntu14 Dev
Indexing Fails to Write to Elasticsearch (nickwallen) closes
apache/metron#1185
Posted by ot...@apache.org.
METRON-1508 In Ubuntu14 Dev Indexing Fails to Write to Elasticsearch (nickwallen) closes apache/metron#1185
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/08fe1cff
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/08fe1cff
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/08fe1cff
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 08fe1cff1430c71205d62e375cbacd6906e4fc4e
Parents: 0d34e77
Author: nickwallen <ni...@nickallen.org>
Authored: Fri Sep 21 15:31:24 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Fri Sep 21 15:31:24 2018 -0400
----------------------------------------------------------------------
.../ansible/roles/ambari_config/vars/single_node_vm.yml | 4 ++--
.../development/centos6/ansible/inventory/group_vars/all | 1 +
.../development/ubuntu14/ansible/inventory/group_vars/all | 1 +
.../ELASTICSEARCH/5.6.2/configuration/elastic-site.xml | 8 ++++----
4 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index 962726c..2ec956a 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -92,7 +92,7 @@ configurations:
# Storm expects ambari metrics to be available in 2.6. We do *not* install ambari metrics in full-dev, so we need to revert to the old consumer
storm.cluster.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer"}]'
topology.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer", "parallelism.hint": 1, "whitelist": ["kafkaOffset\\..+/", "__complete-latency", "__process-latency", "__receive\\.population$", "__sendqueue\\.population$", "__execute-count", "__emit-count", "__ack-count", "__fail-count", "memory/heap\\.usedBytes$", "memory/nonHeap\\.usedBytes$", "GC/.+\\.count$", "GC/.+\\.timeMs$"]}]'
- # Storm expects ambari metrics to be available in 2.6 and ambari metrics pulls data via JMX, but since we don't use ambari metrics here, we don't have the javaagent around to use and thus that must be removed from nimbus, supervisor and worker properties
+ # Storm expects ambari metrics to be available in 2.6 and ambari metrics pulls data via JMX, but since we don't use ambari metrics here, we don't have the javaagent around to use and thus that must be removed from nimbus, supervisor and worker properties
nimbus.childopts: '-Xmx1024m _JAAS_PLACEHOLDER'
supervisor.childopts: '-Xmx256m _JAAS_PLACEHOLDER'
worker.childopts: "-Xmx768m _JAAS_PLACEHOLDER"
@@ -117,7 +117,7 @@ configurations:
required_configurations:
- metron-env:
storm_rest_addr: "http://{{ groups.ambari_slave[0] }}:8744"
- es_hosts: "{{ groups.search | join(',') }}"
+ es_hosts: "{{ elasticsearch_hosts }}"
zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995"
solr_zookeeper_url: "{{ groups.search[0] }}:9983"
- metron-rest-env:
http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/development/centos6/ansible/inventory/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/ansible/inventory/group_vars/all b/metron-deployment/development/centos6/ansible/inventory/group_vars/all
index dcb7b42..550eb8f 100644
--- a/metron-deployment/development/centos6/ansible/inventory/group_vars/all
+++ b/metron-deployment/development/centos6/ansible/inventory/group_vars/all
@@ -46,6 +46,7 @@ pycapa_home: "/opt/pycapa"
snort_version: "2.9.8.0-1"
snort_alert_csv_path: "/var/log/snort/alert.csv"
threat_intel_bulk_load: False
+elasticsearch_hosts: "{{ groups.search | join(',') }}"
# data directories - only required to override defaults
zookeeper_data_dir: "/data1/hadoop/zookeeper"
http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
index ef9f705..1338d7c 100644
--- a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
+++ b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
@@ -46,6 +46,7 @@ pycapa_home: "/opt/pycapa"
snort_version: "2.9.8.0-1"
snort_alert_csv_path: "/var/log/snort/alert.csv"
threat_intel_bulk_load: False
+elasticsearch_hosts: "localhost"
# data directories - only required to override defaults
zookeeper_data_dir: "/data1/hadoop/zookeeper"
http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
index 34df1e4..440c66d 100755
--- a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml
@@ -111,7 +111,7 @@
<value>0</value>
<description>expected_data_nodes</description>
</property>
- <!-- Index -->
+ <!-- Index -->
<property>
<name>index_merge_scheduler_max_thread_count</name>
<value>5</value>
@@ -171,7 +171,7 @@
<name>cluster_routing_allocation_disk_threshold_enabled</name>
<value>true</value>
<description>Property used when multiple drives are used to understand if thresholding is active</description>
- </property>
+ </property>
<property>
<name>cluster_routing_allocation_disk_watermark_low</name>
<value>.97</value>
@@ -185,7 +185,7 @@
<property>
<name>network_host</name>
<value>[ _local_, _site_ ]</value>
- <description>Network interface(s) ES will bind to within each node. "_site_" or a more specific external address is required for all multi-node clusters, and also recommended for single-node installs to allow access to ES reports from non-local hosts. Always include the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description>
+ <description>Network interface(s) ES will bind to within each node. "_site_" or a more specific external address is required for all multi-node clusters, and also recommended for single-node installs to allow access to ES reports from non-local hosts. Always include the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/5.6/modules-network.html for ES documentation.</description>
</property>
<property>
<name>network_publish_host</name>
@@ -193,6 +193,6 @@
<value-attributes>
<empty-value-valid>true</empty-value-valid>
</value-attributes>
- <description>Network address ES will publish for client and peer use. Empty value causes it to pick from the values in network_host, which works in most simple environments. MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description>
+ <description>Network address ES will publish for client and peer use. Empty value causes it to pick from the values in network_host, which works in most simple environments. MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/5.6/modules-network.html for ES documentation.</description>
</property>
</configuration>
[47/51] [abbrv] metron git commit: METRON-1829 Large Error Message
Causes Slow Search Performance (merrimanr) closes apache/metron#1239
Posted by ot...@apache.org.
METRON-1829 Large Error Message Causes Slow Search Performance (merrimanr) closes apache/metron#1239
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d44a3925
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d44a3925
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d44a3925
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: d44a392567e35022bfb35a787b55aff3447ac60e
Parents: 2531c3e
Author: merrimanr <me...@gmail.com>
Authored: Mon Oct 22 08:43:10 2018 -0500
Committer: rmerriman <me...@gmail.com>
Committed: Mon Oct 22 08:43:10 2018 -0500
----------------------------------------------------------------------
.../bolt/BulkMessageWriterBoltTest.java | 2 +-
metron-platform/metron-writer/pom.xml | 6 ++
.../metron/writer/BulkWriterComponent.java | 33 ++++++-----
.../writer/bolt/BulkMessageWriterBolt.java | 2 +-
.../metron/writer/BulkWriterComponentTest.java | 61 +++++++++++++++-----
5 files changed, 76 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/d44a3925/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
index 588fc58..083628c 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
@@ -207,7 +207,7 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
}
UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.ERROR);
verify(outputCollector, times(5)).ack(tuple);
- verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), any(Values.class));
+ verify(outputCollector, times(5)).emit(eq(Constants.ERROR_STREAM), any(Values.class));
verify(outputCollector, times(1)).reportError(any(Throwable.class));
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d44a3925/metron-platform/metron-writer/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml
index e845516..6d08093 100644
--- a/metron-platform/metron-writer/pom.xml
+++ b/metron-platform/metron-writer/pom.xml
@@ -207,6 +207,12 @@
<artifactId>metron-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-test-utilities</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/metron/blob/d44a3925/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
index 7678584..68585c5 100644
--- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
@@ -40,6 +40,7 @@ import org.apache.metron.common.writer.BulkMessageWriter;
import org.apache.metron.common.writer.BulkWriterResponse;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -118,12 +119,18 @@ public class BulkWriterComponent<MESSAGE_T> {
public void error(String sensorType, Throwable e, Iterable<Tuple> tuples, MessageGetStrategy messageGetStrategy) {
LOG.error(format("Failing %d tuple(s); sensorType=%s", Iterables.size(tuples), sensorType), e);
- MetronError error = new MetronError()
- .withSensorType(Collections.singleton(sensorType))
- .withErrorType(Constants.ErrorType.INDEXING_ERROR)
- .withThrowable(e);
- tuples.forEach(t -> error.addRawMessage(messageGetStrategy.get(t)));
- handleError(tuples, error);
+ tuples.forEach(t -> {
+ MetronError error = new MetronError()
+ .withSensorType(Collections.singleton(sensorType))
+ .withErrorType(Constants.ErrorType.INDEXING_ERROR)
+ .withThrowable(e)
+ .addRawMessage(messageGetStrategy.get(t));
+ collector.emit(Constants.ERROR_STREAM, new Values(error.getJSONObject()));
+ collector.ack(t);
+ });
+ // there is only one error to report for all of the failed tuples
+ collector.reportError(e);
+
}
/**
@@ -133,24 +140,24 @@ public class BulkWriterComponent<MESSAGE_T> {
* <p>Without a valid message, the JSON message cannot be added to the error.
*
* @param e The exception that occurred.
- * @param tuples The tuples to error that may not contain valid messages.
+ * @param tuple The tuple to error that may not contain a valid message.
*/
- public void error(Throwable e, Iterable<Tuple> tuples) {
- LOG.error(format("Failing %d tuple(s)", Iterables.size(tuples)), e);
+ public void error(Throwable e, Tuple tuple) {
+ LOG.error("Failing tuple", e);
MetronError error = new MetronError()
.withErrorType(Constants.ErrorType.INDEXING_ERROR)
.withThrowable(e);
- handleError(tuples, error);
+ handleError(tuple, error);
}
/**
* Errors a set of tuples.
*
- * @param tuples The tuples to error.
+ * @param tuple The tuple to error.
* @param error
*/
- private void handleError(Iterable<Tuple> tuples, MetronError error) {
- tuples.forEach(t -> collector.ack(t));
+ private void handleError(Tuple tuple, MetronError error) {
+ collector.ack(tuple);
ErrorUtils.handleError(collector, error);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d44a3925/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
index 4bb3888..590ab8c 100644
--- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
@@ -311,7 +311,7 @@ public class BulkMessageWriterBolt<CONFIG_T extends Configurations> extends Conf
LOG.debug("Unable to extract message from tuple; expected valid JSON");
getWriterComponent().error(
new Exception("Unable to extract message from tuple; expected valid JSON"),
- ImmutableList.of(tuple)
+ tuple
);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d44a3925/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java
index c389854..decf3a5 100644
--- a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java
+++ b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java
@@ -18,10 +18,12 @@
package org.apache.metron.writer;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.argThat;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
import static org.powermock.api.mockito.PowerMockito.mockStatic;
import static org.powermock.api.mockito.PowerMockito.verifyStatic;
@@ -36,8 +38,10 @@ import org.apache.metron.common.message.MessageGetStrategy;
import org.apache.metron.common.utils.ErrorUtils;
import org.apache.metron.common.writer.BulkMessageWriter;
import org.apache.metron.common.writer.BulkWriterResponse;
+import org.apache.metron.test.error.MetronErrorJSONMatcher;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
import org.json.simple.JSONObject;
import org.junit.Before;
import org.junit.Rule;
@@ -127,9 +131,12 @@ public class BulkWriterComponentTest {
@Test
public void writeShouldProperlyHandleWriterErrors() throws Exception {
Throwable e = new Exception("test exception");
- MetronError error = new MetronError()
+ MetronError expectedError1 = new MetronError()
.withSensorType(Collections.singleton(sensorType))
- .withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Arrays.asList(message1, message2));
+ .withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Collections.singletonList(message1));
+ MetronError expectedError2 = new MetronError()
+ .withSensorType(Collections.singleton(sensorType))
+ .withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Collections.singletonList(message2));
BulkWriterResponse response = new BulkWriterResponse();
response.addAllErrors(e, tupleList);
@@ -139,8 +146,14 @@ public class BulkWriterComponentTest {
bulkWriterComponent.write(sensorType, tuple1, message1, bulkMessageWriter, configurations, messageGetStrategy);
bulkWriterComponent.write(sensorType, tuple2, message2, bulkMessageWriter, configurations, messageGetStrategy);
- verifyStatic(times(1));
- ErrorUtils.handleError(collector, error);
+ verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM),
+ new Values(argThat(new MetronErrorJSONMatcher(expectedError1.getJSONObject()))));
+ verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM),
+ new Values(argThat(new MetronErrorJSONMatcher(expectedError2.getJSONObject()))));
+ verify(collector, times(1)).ack(tuple1);
+ verify(collector, times(1)).ack(tuple2);
+ verify(collector, times(1)).reportError(e);
+ verifyNoMoreInteractions(collector);
}
@Test
@@ -161,9 +174,16 @@ public class BulkWriterComponentTest {
@Test
public void writeShouldProperlyHandleWriterException() throws Exception {
Throwable e = new Exception("test exception");
- MetronError error = new MetronError()
+ MetronError expectedError1 = new MetronError()
+ .withSensorType(Collections.singleton(sensorType))
+ .withErrorType(Constants.ErrorType.INDEXING_ERROR)
+ .withThrowable(e)
+ .withRawMessages(Collections.singletonList(message1));
+ MetronError expectedError2 = new MetronError()
.withSensorType(Collections.singleton(sensorType))
- .withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Arrays.asList(message1, message2));
+ .withErrorType(Constants.ErrorType.INDEXING_ERROR)
+ .withThrowable(e)
+ .withRawMessages(Collections.singletonList(message2));
BulkWriterResponse response = new BulkWriterResponse();
response.addAllErrors(e, tupleList);
@@ -173,8 +193,14 @@ public class BulkWriterComponentTest {
bulkWriterComponent.write(sensorType, tuple1, message1, bulkMessageWriter, configurations, messageGetStrategy);
bulkWriterComponent.write(sensorType, tuple2, message2, bulkMessageWriter, configurations, messageGetStrategy);
- verifyStatic(times(1));
- ErrorUtils.handleError(collector, error);
+ verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM),
+ new Values(argThat(new MetronErrorJSONMatcher(expectedError1.getJSONObject()))));
+ verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM),
+ new Values(argThat(new MetronErrorJSONMatcher(expectedError2.getJSONObject()))));
+ verify(collector, times(1)).ack(tuple1);
+ verify(collector, times(1)).ack(tuple2);
+ verify(collector, times(1)).reportError(e);
+ verifyNoMoreInteractions(collector);
}
@Test
@@ -182,19 +208,28 @@ public class BulkWriterComponentTest {
Throwable e = new Exception("test exception");
MetronError error1 = new MetronError()
.withSensorType(Collections.singleton("sensor1"))
- .withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Collections.singletonList(message1));
+ .withErrorType(Constants.ErrorType.INDEXING_ERROR)
+ .withThrowable(e)
+ .withRawMessages(Collections.singletonList(message1));
MetronError error2 = new MetronError()
.withSensorType(Collections.singleton("sensor2"))
- .withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Collections.singletonList(message2));
+ .withErrorType(Constants.ErrorType.INDEXING_ERROR)
+ .withThrowable(e)
+ .withRawMessages(Collections.singletonList(message2));
BulkWriterComponent<JSONObject> bulkWriterComponent = new BulkWriterComponent<>(collector);
bulkWriterComponent.write("sensor1", tuple1, message1, bulkMessageWriter, configurations, messageGetStrategy);
bulkWriterComponent.write("sensor2", tuple2, message2, bulkMessageWriter, configurations, messageGetStrategy);
bulkWriterComponent.errorAll(e, messageGetStrategy);
- verifyStatic(times(1));
- ErrorUtils.handleError(collector, error1);
- ErrorUtils.handleError(collector, error2);
+ verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM),
+ new Values(argThat(new MetronErrorJSONMatcher(error1.getJSONObject()))));
+ verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM),
+ new Values(argThat(new MetronErrorJSONMatcher(error2.getJSONObject()))));
+ verify(collector, times(1)).ack(tuple1);
+ verify(collector, times(1)).ack(tuple2);
+ verify(collector, times(2)).reportError(e);
+ verifyNoMoreInteractions(collector);
bulkWriterComponent.write("sensor1", tuple1, message1, bulkMessageWriter, configurations, messageGetStrategy);
verify(bulkMessageWriter, times(0)).write(sensorType, configurations, Collections.singletonList(tuple1), Collections.singletonList(message1));
[14/51] [abbrv] metron git commit: METRON-1787 Input Time Constraints
for Batch Profiler (nickwallen) closes apache/metron#1209
Posted by ot...@apache.org.
METRON-1787 Input Time Constraints for Batch Profiler (nickwallen) closes apache/metron#1209
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fa3be8d3
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fa3be8d3
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fa3be8d3
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: fa3be8d32ccadcd11edad046cbd063cec3a20624
Parents: 1545978
Author: nickwallen <ni...@nickallen.org>
Authored: Wed Sep 26 18:13:30 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Wed Sep 26 18:13:30 2018 -0400
----------------------------------------------------------------------
.../clock/EventTimeOnlyClockFactory.java | 58 ++++++++
.../clock/EventTimeOnlyClockFactoryTest.java | 61 +++++++++
.../metron-profiler-spark/README.md | 19 +++
.../metron/profiler/spark/BatchProfiler.java | 38 +++++-
.../profiler/spark/BatchProfilerConfig.java | 6 +-
.../metron/profiler/spark/TimestampParser.java | 55 ++++++++
.../spark/function/MessageRouterFunction.java | 106 +++++++++++++--
.../spark/BatchProfilerIntegrationTest.java | 58 +++++++-
.../profiler/spark/TimestampParserTest.java | 67 ++++++++++
.../function/MessageRouterFunctionTest.java | 133 +++++++++++++++++--
10 files changed, 579 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/fa3be8d3/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactory.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactory.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactory.java
new file mode 100644
index 0000000..2f9ca7c
--- /dev/null
+++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.clock;
+
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
+
+import java.io.Serializable;
+
+/**
+ * Creates a {@link Clock} based on the profiler configuration. This should
+ * be used in cases where only event time is accceptable.
+ *
+ * <p>If the Profiler is configured to use event time, a {@link EventTimeClock} will
+ * be created. Otherwise, an {@link IllegalStateException} is thrown.
+ */
+public class EventTimeOnlyClockFactory implements ClockFactory, Serializable {
+
+ /**
+ * If the Profiler is configured to use event time, a {@link EventTimeClock} is created.
+ * Otherwise, an {@link IllegalArgumentException} is thrown.
+ *
+ * @param config The profiler configuration.
+ * @return The appropriate Clock based on the profiler configuration.
+ * @throws IllegalStateException If the profiler configuration is set to system time.
+ */
+ @Override
+ public Clock createClock(ProfilerConfig config) {
+ Clock clock;
+
+ boolean isEventTime = config.getTimestampField().isPresent();
+ if(isEventTime) {
+ String timestampField = config.getTimestampField().get();
+ clock = new EventTimeClock(timestampField);
+
+ } else {
+ throw new IllegalStateException("Expected profiler to use event time.");
+ }
+
+ return clock;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/fa3be8d3/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactoryTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactoryTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactoryTest.java
new file mode 100644
index 0000000..f1d4114
--- /dev/null
+++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/clock/EventTimeOnlyClockFactoryTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.clock;
+
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Optional;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests the {@link EventTimeOnlyClockFactory}.
+ */
+public class EventTimeOnlyClockFactoryTest {
+
+ private EventTimeOnlyClockFactory clockFactory;
+
+ @Before
+ public void setup() {
+ clockFactory = new EventTimeOnlyClockFactory();
+ }
+
+ @Test
+ public void testCreateEventTimeClock() {
+ // configure the profiler to use event time
+ ProfilerConfig config = new ProfilerConfig();
+ config.setTimestampField(Optional.of("timestamp"));
+
+ // the factory should return a clock that handles 'event time'
+ Clock clock = clockFactory.createClock(config);
+ assertTrue(clock instanceof EventTimeClock);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testCreateProcessingTimeClock() {
+ // the profiler uses processing time by default
+ ProfilerConfig config = new ProfilerConfig();
+ clockFactory.createClock(config);
+ fail("Expected exception");
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/fa3be8d3/metron-analytics/metron-profiler-spark/README.md
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/README.md b/metron-analytics/metron-profiler-spark/README.md
index 99e8c7e..df143f1 100644
--- a/metron-analytics/metron-profiler-spark/README.md
+++ b/metron-analytics/metron-profiler-spark/README.md
@@ -245,6 +245,8 @@ You can store both settings for the Profiler along with settings for Spark in th
|--- |---
| [`profiler.batch.input.path`](#profilerbatchinputpath) | The path to the input data read by the Batch Profiler.
| [`profiler.batch.input.format`](#profilerbatchinputformat) | The format of the input data read by the Batch Profiler.
+| [`profiler.batch.input.begin`](#profilerbatchinputend) | Only messages with a timestamp after this will be profiled.
+| [`profiler.batch.input.end`](#profilerbatchinputbegin) | Only messages with a timestamp before this will be profiled.
| [`profiler.period.duration`](#profilerperiodduration) | The duration of each profile period.
| [`profiler.period.duration.units`](#profilerperioddurationunits) | The units used to specify the [`profiler.period.duration`](#profilerperiodduration).
| [`profiler.hbase.salt.divisor`](#profilerhbasesaltdivisor) | A salt is prepended to the row key to help prevent hot-spotting.
@@ -263,6 +265,23 @@ The path to the input data read by the Batch Profiler.
The format of the input data read by the Batch Profiler.
+### `profiler.batch.input.begin`
+
+*Default*: undefined; no time constraint
+
+Only messages with a timestamp equal to or after this will be profiled. The Profiler will only profiles messages with a timestamp in [`profiler.batch.input.begin`, `profiler.batch.input.end`] inclusive.
+
+By default, no time constraint is defined. The value is expected to follow the [ISO-8601 instant format](https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html#ISO_INSTANT); 2011-12-03T10:15:30Z.
+
+
+### `profiler.batch.input.end`
+
+*Default*: undefined; no time constraint
+
+Only messages with a timestamp before or equal to this will be profiled. The Profiler will only profiles messages with a timestamp in [`profiler.batch.input.begin`, `profiler.batch.input.end`] inclusive.
+
+By default, no time constraint is defined. The value is expected to follow the [ISO-8601 instant format](https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html#ISO_INSTANT); 2011-12-03T10:15:30Z.
+
### `profiler.period.duration`
*Default*: 15
http://git-wip-us.apache.org/repos/asf/metron/blob/fa3be8d3/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
index d75abc3..39f8b3a 100644
--- a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
@@ -34,10 +34,12 @@ import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.lang.invoke.MethodHandles;
-import java.util.HashMap;
import java.util.Map;
+import java.util.Optional;
import java.util.Properties;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_BEGIN;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_END;
import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
import static org.apache.spark.sql.functions.sum;
@@ -51,6 +53,12 @@ public class BatchProfiler implements Serializable {
protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private TimestampParser timestampParser;
+
+ public BatchProfiler() {
+ this.timestampParser = new TimestampParser();
+ }
+
/**
* Execute the Batch Profiler.
*
@@ -69,7 +77,6 @@ public class BatchProfiler implements Serializable {
LOG.debug("Building {} profile(s)", profiles.getProfiles().size());
Map<String, String> globals = Maps.fromProperties(globalProperties);
-
String inputFormat = TELEMETRY_INPUT_FORMAT.get(profilerProps, String.class);
String inputPath = TELEMETRY_INPUT_PATH.get(profilerProps, String.class);
LOG.debug("Loading telemetry from '{}'", inputPath);
@@ -85,7 +92,7 @@ public class BatchProfiler implements Serializable {
// find all routes for each message
Dataset<MessageRoute> routes = telemetry
- .flatMap(new MessageRouterFunction(profiles, globals), Encoders.bean(MessageRoute.class));
+ .flatMap(messageRouterFunction(profilerProps, profiles, globals), Encoders.bean(MessageRoute.class));
LOG.debug("Generated {} message route(s)", routes.cache().count());
// build the profiles
@@ -104,4 +111,29 @@ public class BatchProfiler implements Serializable {
return count;
}
+
+ /**
+ * Builds the function that performs message routing.
+ *
+ * @param profilerProps The profiler configuration properties.
+ * @param profiles The profile definitions.
+ * @param globals The Stellar global properties.
+ * @return A {@link MessageRouterFunction}.
+ */
+ private MessageRouterFunction messageRouterFunction(
+ Properties profilerProps,
+ ProfilerConfig profiles,
+ Map<String, String> globals) {
+ MessageRouterFunction routerFunction = new MessageRouterFunction(profiles, globals);
+
+ // an optional time constraint to limit how far back to look for telemetry
+ Optional<Long> beginAt = timestampParser.parse(TELEMETRY_INPUT_BEGIN.get(profilerProps, String.class));
+ beginAt.ifPresent(begin -> routerFunction.withBegin(begin));
+
+ // an optional time constraint to limit the most recent telemetry
+ Optional<Long> endAt = timestampParser.parse(TELEMETRY_INPUT_END.get(profilerProps, String.class));
+ endAt.ifPresent(end -> routerFunction.withEnd(end));
+
+ return routerFunction;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/fa3be8d3/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
index 054806e..e8cd160 100644
--- a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
@@ -46,7 +46,11 @@ public enum BatchProfilerConfig {
TELEMETRY_INPUT_FORMAT("profiler.batch.input.format", "text", String.class),
- TELEMETRY_INPUT_PATH("profiler.batch.input.path", "hdfs://localhost:9000/apps/metron/indexing/indexed/*/*", String.class);
+ TELEMETRY_INPUT_PATH("profiler.batch.input.path", "hdfs://localhost:9000/apps/metron/indexing/indexed/*/*", String.class),
+
+ TELEMETRY_INPUT_BEGIN("profiler.batch.input.begin", "", String.class),
+
+ TELEMETRY_INPUT_END("profiler.batch.input.end", "", String.class);
/**
* The key for the configuration value.
http://git-wip-us.apache.org/repos/asf/metron/blob/fa3be8d3/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/TimestampParser.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/TimestampParser.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/TimestampParser.java
new file mode 100644
index 0000000..4be8b3e
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/TimestampParser.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.spark;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.util.Optional;
+
+/**
+ * Parses an input string and returns a timestamp in epoch milliseconds.
+ */
+public class TimestampParser {
+
+ /**
+ * Parses an input string and returns an optional timestamp in epoch milliseconds.
+ *
+ * @param inputString The input defining a timestamp.
+ * @return A timestamp in epoch milliseconds.
+ */
+ public Optional<Long> parse(String inputString) {
+ Optional<Long> epochMilli = Optional.empty();
+
+ // a blank is acceptable and treated as undefined
+ if (StringUtils.isNotBlank(inputString)) {
+ epochMilli = Optional.of(new DateTimeFormatterBuilder()
+ .append(DateTimeFormatter.ISO_INSTANT)
+ .toFormatter()
+ .parse(inputString, Instant::from)
+ .toEpochMilli());
+ }
+
+ return epochMilli;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/fa3be8d3/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/MessageRouterFunction.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/MessageRouterFunction.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/MessageRouterFunction.java
index cf8029f..31734d0 100644
--- a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/MessageRouterFunction.java
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/MessageRouterFunction.java
@@ -23,7 +23,11 @@ import org.apache.metron.common.configuration.profiler.ProfilerConfig;
import org.apache.metron.profiler.DefaultMessageRouter;
import org.apache.metron.profiler.MessageRoute;
import org.apache.metron.profiler.MessageRouter;
+import org.apache.metron.profiler.clock.Clock;
+import org.apache.metron.profiler.clock.ClockFactory;
+import org.apache.metron.profiler.clock.EventTimeOnlyClockFactory;
import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.StellarFunctions;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
@@ -54,9 +58,27 @@ public class MessageRouterFunction implements FlatMapFunction<String, MessageRou
*/
private ProfilerConfig profilerConfig;
+ /**
+ * A clock that can extract time from the messages received.
+ */
+ private Clock clock;
+
+ /**
+ * Only messages with a timestamp after this will be routed.
+ */
+ private Long begin;
+
+ /**
+ * Only messages with a timestamp before this will be routed.
+ */
+ private Long end;
+
public MessageRouterFunction(ProfilerConfig profilerConfig, Map<String, String> globals) {
this.profilerConfig = profilerConfig;
this.globals = globals;
+ this.begin = Long.MIN_VALUE;
+ this.end = Long.MAX_VALUE;
+ withClockFactory(new EventTimeOnlyClockFactory());
}
/**
@@ -70,8 +92,7 @@ public class MessageRouterFunction implements FlatMapFunction<String, MessageRou
*/
@Override
public Iterator<MessageRoute> call(String jsonMessage) throws Exception {
- List<MessageRoute> routes;
-
+ List<MessageRoute> routes = Collections.emptyList();
JSONParser parser = new JSONParser();
Context context = TaskUtils.getContext(globals);
MessageRouter router = new DefaultMessageRouter(context);
@@ -80,20 +101,67 @@ public class MessageRouterFunction implements FlatMapFunction<String, MessageRou
Optional<JSONObject> message = toMessage(jsonMessage, parser);
if(message.isPresent()) {
- // find all routes
- routes = router.route(message.get(), profilerConfig, context);
- LOG.trace("Found {} route(s) for a message", routes.size());
+ // extract the timestamp from the message
+ Optional<Long> timestampOpt = clock.currentTimeMillis(message.get());
+ if (timestampOpt.isPresent()) {
+
+ // timestamp must be in [begin, end]
+ Long timestamp = timestampOpt.get();
+ if(timestamp >= begin && timestamp <= end) {
+ routes = router.route(message.get(), profilerConfig, context);
+ LOG.trace("Found {} route(s) for a message", routes.size());
+
+ } else {
+ LOG.trace("Ignoring message; timestamp={} not in [{},{}]", timestamp, prettyPrint(begin), prettyPrint(end));
+ }
+
+ } else {
+ LOG.trace("No timestamp in message. Message will be ignored.");
+ }
} else {
- // the message is not valid and must be ignored
- routes = Collections.emptyList();
- LOG.trace("No route possible. Unable to parse message.");
+ LOG.trace("Unable to parse message. Message will be ignored");
}
return routes.iterator();
}
/**
+ * Set a time constraint.
+ *
+ * @param begin Only messages with a timestamp after this will be routed.
+ * @return The message router function
+ */
+ public MessageRouterFunction withBegin(Long begin) {
+ this.begin = begin;
+ return this;
+ }
+
+ /**
+ * Set a time constraint.
+ *
+ * @param end Only messages with a timestamp before this will be routed.
+ * @return The message router function
+ */
+ public MessageRouterFunction withEnd(Long end) {
+ this.end = end;
+ return this;
+ }
+
+ /**
+ * Defines the {@link ClockFactory} used to create the {@link Clock}.
+ *
+ * <p>Calling this method is only needed to override the default behavior.
+ *
+ * @param clockFactory The factory to use for creating the {@link Clock}.
+ * @return The message router function.
+ */
+ public MessageRouterFunction withClockFactory(ClockFactory clockFactory) {
+ this.clock = clockFactory.createClock(profilerConfig);
+ return this;
+ }
+
+ /**
* Parses the raw JSON of a message.
*
* @param json The raw JSON to parse.
@@ -110,4 +178,26 @@ public class MessageRouterFunction implements FlatMapFunction<String, MessageRou
return Optional.empty();
}
}
+
+ /**
+ * Pretty prints a Long value for use when logging these values.
+ *
+ * <p>Long.MIN_VALUE and Long.MAX_VALUE will occur frequently and is difficult to grok in the logs. Instead
+ * Long.MIN_VALUE is rendered as "MIN". Long.MAX_VALUE is srendered as "MAX". All other values are rendered
+ * directly.
+ *
+ * @param value The value to pretty print.
+ * @return
+ */
+ private static String prettyPrint(Long value) {
+ String result;
+ if(value == Long.MIN_VALUE) {
+ result = "MIN";
+ } else if(value == Long.MAX_VALUE) {
+ result = "MAX";
+ } else {
+ result = value.toString();
+ }
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/fa3be8d3/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
index 87c4246..c33644f 100644
--- a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
@@ -38,8 +38,11 @@ import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -51,6 +54,8 @@ import static org.apache.metron.profiler.client.stellar.ProfilerClientConfig.PRO
import static org.apache.metron.profiler.spark.BatchProfilerConfig.HBASE_COLUMN_FAMILY;
import static org.apache.metron.profiler.spark.BatchProfilerConfig.HBASE_TABLE_NAME;
import static org.apache.metron.profiler.spark.BatchProfilerConfig.HBASE_TABLE_PROVIDER;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_BEGIN;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_END;
import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
import static org.junit.Assert.assertTrue;
@@ -60,6 +65,7 @@ import static org.junit.Assert.assertTrue;
*/
public class BatchProfilerIntegrationTest {
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
/**
* {
* "timestampField": "timestamp",
@@ -212,6 +218,54 @@ public class BatchProfilerIntegrationTest {
validateProfiles();
}
+ @Test
+ public void testBatchProfilerWithEndTimeConstraint() throws Exception {
+ // the input telemetry is text/json stored in the local filesystem
+ profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), "src/test/resources/telemetry.json");
+ profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "text");
+
+ // there are 40 messages before "2018-07-07T15:51:48Z" in the test data
+ profilerProperties.put(TELEMETRY_INPUT_BEGIN.getKey(), "");
+ profilerProperties.put(TELEMETRY_INPUT_END.getKey(), "2018-07-07T15:51:48Z");
+
+ BatchProfiler profiler = new BatchProfiler();
+ profiler.run(spark, profilerProperties, getGlobals(), readerProperties, getProfile());
+
+ // the max timestamp in the data is around July 7, 2018
+ assign("maxTimestamp", "1530978728982L");
+
+ // the 'window' looks up to 5 hours before the max timestamp
+ assign("window", "PROFILE_WINDOW('from 5 hours ago', maxTimestamp)");
+
+ assertTrue(execute("[12] == PROFILE_GET('count-by-ip', '192.168.66.1', window)", Boolean.class));
+ assertTrue(execute("[28] == PROFILE_GET('count-by-ip', '192.168.138.158', window)", Boolean.class));
+ assertTrue(execute("[40] == PROFILE_GET('total-count', 'total', window)", Boolean.class));
+ }
+
+ @Test
+ public void testBatchProfilerWithBeginTimeConstraint() throws Exception {
+ // the input telemetry is text/json stored in the local filesystem
+ profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), "src/test/resources/telemetry.json");
+ profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "text");
+
+ // there are 60 messages after "2018-07-07T15:51:48Z" in the test data
+ profilerProperties.put(TELEMETRY_INPUT_BEGIN.getKey(), "2018-07-07T15:51:48Z");
+ profilerProperties.put(TELEMETRY_INPUT_END.getKey(), "");
+
+ BatchProfiler profiler = new BatchProfiler();
+ profiler.run(spark, profilerProperties, getGlobals(), readerProperties, getProfile());
+
+ // the max timestamp in the data is around July 7, 2018
+ assign("maxTimestamp", "1530978728982L");
+
+ // the 'window' looks up to 5 hours before the max timestamp
+ assign("window", "PROFILE_WINDOW('from 5 hours ago', maxTimestamp)");
+
+ assertTrue(execute("[14] == PROFILE_GET('count-by-ip', '192.168.66.1', window)", Boolean.class));
+ assertTrue(execute("[46] == PROFILE_GET('count-by-ip', '192.168.138.158', window)", Boolean.class));
+ assertTrue(execute("[60] == PROFILE_GET('total-count', 'total', window)", Boolean.class));
+ }
+
/**
* Validates the profiles that were built.
*
@@ -263,6 +317,8 @@ public class BatchProfilerIntegrationTest {
* @return The result of executing the Stellar expression.
*/
private <T> T execute(String expression, Class<T> clazz) {
- return executor.execute(expression, Collections.emptyMap(), clazz);
+ T results = executor.execute(expression, Collections.emptyMap(), clazz);
+ LOG.debug("{} = {}", expression, results);
+ return results;
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/fa3be8d3/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/TimestampParserTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/TimestampParserTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/TimestampParserTest.java
new file mode 100644
index 0000000..f760b35
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/TimestampParserTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.spark;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.format.DateTimeParseException;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TimestampParserTest {
+
+ private TimestampParser parser;
+
+ @Before
+ public void setup() {
+ parser = new TimestampParser();
+ }
+
+ @Test
+ public void testEmpty() {
+ Optional<Long> millis = parser.parse("");
+ assertFalse(millis.isPresent());
+ }
+
+ @Test
+ public void testBlank() {
+ Optional<Long> millis = parser.parse(" ");
+ assertFalse(millis.isPresent());
+ }
+
+ @Test
+ public void testIsoInstantFormat() {
+ // ISO-8601 instant format
+ Optional<Long> millis = parser.parse("2011-12-03T10:15:30Z");
+ assertTrue(millis.isPresent());
+ assertEquals(1322907330000L, millis.get().longValue());
+ }
+
+ @Test(expected = DateTimeParseException.class)
+ public void testInvalidFormat() {
+ parser.parse("1537502400000");
+ fail("Expected exception");
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/fa3be8d3/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/MessageRouterFunctionTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/MessageRouterFunctionTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/MessageRouterFunctionTest.java
index ceaa7cd..9a2cbf4 100644
--- a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/MessageRouterFunctionTest.java
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/MessageRouterFunctionTest.java
@@ -31,14 +31,17 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
/**
* Tests the {@link MessageRouterFunction}.
*/
public class MessageRouterFunctionTest {
+ private Long messageTimestamp = 1537468508360L;
+
/**
- * { "ip_src_addr": "192.168.1.22" }
+ * { "ip_src_addr": "192.168.1.22", "timestamp": 1537468508360 }
*/
@Multiline
private String goodMessage;
@@ -48,9 +51,15 @@ public class MessageRouterFunctionTest {
*/
private String badMessage;
+ /**
+ * { "ip_src_addr": "192.168.1.22" }
+ */
+ @Multiline
+ private String messageNoTimestamp;
+
@Test
public void testFindRoutes() throws Exception {
- MessageRouterFunction function = new MessageRouterFunction(oneProfile(), getGlobals());
+ MessageRouterFunction function = new MessageRouterFunction(profile(), getGlobals());
Iterator<MessageRoute> iter = function.call(goodMessage);
List<MessageRoute> routes = Lists.newArrayList(iter);
@@ -58,14 +67,20 @@ public class MessageRouterFunctionTest {
Assert.assertEquals("profile1", routes.get(0).getProfileDefinition().getProfile());
}
- /**
- * A bad or invalid message should return no routes.
- */
+ @Test(expected = IllegalStateException.class)
+ public void testWithSystemTime() throws Exception {
+ MessageRouterFunction function = new MessageRouterFunction(profileWithSystemTime(), getGlobals());
+ Iterator<MessageRoute> iter = function.call(goodMessage);
+
+ Assert.fail("Exception expected as system time is not supported.");
+ }
+
@Test
public void testWithBadMessage() throws Exception {
- MessageRouterFunction function = new MessageRouterFunction(oneProfile(), getGlobals());
+ MessageRouterFunction function = new MessageRouterFunction(profile(), getGlobals());
Iterator<MessageRoute> iter = function.call(badMessage);
+ // an invalid message should return no routes
List<MessageRoute> routes = Lists.newArrayList(iter);
Assert.assertEquals(0, routes.size());
}
@@ -81,7 +96,88 @@ public class MessageRouterFunctionTest {
Assert.assertEquals("profile2", routes.get(1).getProfileDefinition().getProfile());
}
- private ProfilerConfig oneProfile() {
+ @Test
+ public void testWithNoTimestampInMessage() throws Exception {
+ MessageRouterFunction function = new MessageRouterFunction(profile(), getGlobals());
+ Iterator<MessageRoute> iter = function.call(messageNoTimestamp);
+
+ // with no timestamp, the message should be ignored
+ List<MessageRoute> routes = Lists.newArrayList(iter);
+ Assert.assertEquals(0, routes.size());
+ }
+
+ @Test
+ public void testMessageFilteredByBegin() throws Exception {
+ MessageRouterFunction function = new MessageRouterFunction(profile(), getGlobals())
+ .withBegin(messageTimestamp + 1000);
+ Iterator<MessageRoute> iter = function.call(goodMessage);
+
+ // the message should be filtered because it is before `beginAt`
+ List<MessageRoute> routes = Lists.newArrayList(iter);
+ Assert.assertEquals(0, routes.size());
+ }
+
+ @Test
+ public void testMessageNotFilteredByBegin() throws Exception {
+ MessageRouterFunction function = new MessageRouterFunction(profile(), getGlobals())
+ .withBegin(messageTimestamp - 1000);
+ Iterator<MessageRoute> iter = function.call(goodMessage);
+
+ // the message should NOT be filtered because it is after 'beginAt'
+ List<MessageRoute> routes = Lists.newArrayList(iter);
+ Assert.assertEquals(1, routes.size());
+ }
+
+ @Test
+ public void testMessageFilteredByEnd() throws Exception {
+ MessageRouterFunction function = new MessageRouterFunction(profile(), getGlobals())
+ .withEnd(messageTimestamp - 1000);
+ Iterator<MessageRoute> iter = function.call(goodMessage);
+
+ // the message should be filtered because it is after 'endAt'
+ List<MessageRoute> routes = Lists.newArrayList(iter);
+ Assert.assertEquals(0, routes.size());
+ }
+
+ @Test
+ public void testMessageNotFilteredByEnd() throws Exception {
+ MessageRouterFunction function = new MessageRouterFunction(profile(), getGlobals())
+ .withEnd(messageTimestamp + 1000);
+ Iterator<MessageRoute> iter = function.call(goodMessage);
+
+ // the message should NOT be filtered because it is before 'endAt'
+ List<MessageRoute> routes = Lists.newArrayList(iter);
+ Assert.assertEquals(1, routes.size());
+ }
+
+ @Test
+ public void testMessageFilteredByBeginAndEnd() throws Exception {
+ MessageRouterFunction function = new MessageRouterFunction(profile(), getGlobals())
+ .withBegin(messageTimestamp + 1000)
+ .withEnd(messageTimestamp + 2000);
+ Iterator<MessageRoute> iter = function.call(goodMessage);
+
+ // the message should be filtered because it is outside of [beginAt, endAt]
+ List<MessageRoute> routes = Lists.newArrayList(iter);
+ Assert.assertEquals(0, routes.size());
+ }
+
+ @Test
+ public void testMessageNotFilteredByBeginAndEnd() throws Exception {
+ MessageRouterFunction function = new MessageRouterFunction(profile(), getGlobals())
+ .withBegin(messageTimestamp - 1000)
+ .withEnd(messageTimestamp + 1000);
+ Iterator<MessageRoute> iter = function.call(goodMessage);
+
+ // the message should NOT be filtered because it is after 'endAt'
+ List<MessageRoute> routes = Lists.newArrayList(iter);
+ Assert.assertEquals(1, routes.size());
+ }
+
+ /**
+ * Creates a profiler definition, using event time, and containing one profile.
+ */
+ private ProfilerConfig profile() {
ProfileConfig profile = new ProfileConfig()
.withProfile("profile1")
.withForeach("ip_src_addr")
@@ -89,9 +185,13 @@ public class MessageRouterFunctionTest {
.withResult("count");
return new ProfilerConfig()
- .withProfile(profile);
+ .withProfile(profile)
+ .withTimestampField(Optional.of("timestamp"));
}
+ /**
+ * Creates a profiler definition, using event time, and containing two profiles.
+ */
private ProfilerConfig twoProfiles() {
ProfileConfig profile1 = new ProfileConfig()
.withProfile("profile1")
@@ -105,7 +205,22 @@ public class MessageRouterFunctionTest {
.withResult("count");
return new ProfilerConfig()
.withProfile(profile1)
- .withProfile(profile2);
+ .withProfile(profile2)
+ .withTimestampField(Optional.of("timestamp"));
+ }
+
+ /**
+ * Creates a profiler definition using system time.
+ */
+ private ProfilerConfig profileWithSystemTime() {
+ ProfileConfig profile = new ProfileConfig()
+ .withProfile("profile1")
+ .withForeach("ip_src_addr")
+ .withUpdate("count", "count + 1")
+ .withResult("count");
+
+ return new ProfilerConfig()
+ .withProfile(profile);
}
private Map<String, String> getGlobals() {
[10/51] [abbrv] metron git commit: METRON-1759 PCAP UI: Removing
wrong Input annotations from pcap panel component (tiborm via nickwallen)
closes apache/metron#1180
Posted by ot...@apache.org.
METRON-1759 PCAP UI: Removing wrong Input annotations from pcap panel component (tiborm via nickwallen) closes apache/metron#1180
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3372aa36
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3372aa36
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3372aa36
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 3372aa360315b537ee675ecb7d79fcd6885ba13c
Parents: b84c874
Author: tiborm <ti...@gmail.com>
Authored: Wed Sep 19 10:39:38 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Wed Sep 19 10:39:38 2018 -0400
----------------------------------------------------------------------
.../src/app/pcap/pcap-panel/pcap-panel.component.ts | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/3372aa36/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
index 8e4c375..f767020 100644
--- a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
@@ -32,9 +32,9 @@ import { RestError } from '../../model/rest-error';
})
export class PcapPanelComponent implements OnInit, OnDestroy {
- @Input() pdml: Pdml = null;
- @Input() pcapRequest: PcapRequest;
- @Input() resetPaginationForSearch: boolean;
+ pdml: Pdml = null;
+ pcapRequest: PcapRequest;
+ resetPaginationForSearch: boolean;
statusSubscription: Subscription;
cancelSubscription: Subscription;
[34/51] [abbrv] metron git commit: METRON-1812: Fix
dependencies_with_url.csv (mmiklavc via mmiklavc) closes apache/metron#1230
Posted by ot...@apache.org.
METRON-1812: Fix dependencies_with_url.csv (mmiklavc via mmiklavc) closes apache/metron#1230
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1f1b9cdd
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1f1b9cdd
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1f1b9cdd
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 1f1b9cdd3cef7557ff68b7ab33693161b142ef1d
Parents: 747220f
Author: mmiklavc <mi...@gmail.com>
Authored: Tue Oct 9 14:51:57 2018 -0600
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Tue Oct 9 14:51:57 2018 -0600
----------------------------------------------------------------------
dependencies_with_url.csv | 32 ++++++++++++++------------------
1 file changed, 14 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/1f1b9cdd/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 53977f3..fb6c03c 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -256,12 +256,8 @@ io.dropwizard.metrics:metrics-json:jar:3.1.5:compile,ASLv2,https://github.com/dr
io.dropwizard.metrics:metrics-jvm:jar:3.1.5:compile,ASLv2,https://github.com/dropwizard/metrics
io.netty:netty-all:jar:4.0.23.Final:compile,ASLv2,
io.netty:netty-all:jar:4.0.23.Final:provided,ASLv2,
-<<<<<<< HEAD
io.netty:netty-all:jar:4.1.17.Final:compile,ASLv2,
-=======
io.netty:netty-all:jar:4.1.23.Final:compile,ASLv2,
-io.netty:netty:jar:3.10.5.Final:compile,Apache License, Version 2.0,http://netty.io/
->>>>>>> apache/master
io.netty:netty:jar:3.6.2.Final:compile,Apache License, Version 2.0,http://netty.io/
io.netty:netty:jar:3.7.0.Final:compile,Apache License, Version 2.0,http://netty.io/
io.netty:netty:jar:3.9.9.Final:compile,Apache License, Version 2.0,http://netty.io/
@@ -472,20 +468,20 @@ org.eclipse.persistence:org.eclipse.persistence.jpa:jar:2.6.4:compile,EPL 1.0,ht
com.github.ben-manes.caffeine:caffeine:jar:2.6.2:compile,ASLv2,https://github.com/ben-manes/caffeine/blob/v2.6.2/LICENSE
com.google.code.gson:gson:jar:2.2:compile,ASLv2,https://github.com/google/gson
com.google.code.gson:gson:jar:2.8.2:compile,ASLv2,https://github.com/google/gson
- org.codehaus.plexus:plexus-classworlds:jar:2.4:compile
- org.codehaus.plexus:plexus-component-annotations:jar:1.5.5:compile
- org.codehaus.plexus:plexus-interpolation:jar:1.14:compile
- org.codehaus.plexus:plexus-utils:jar:2.0.7:compile
- org.jsoup:jsoup:jar:1.6.1:compile
- org.sonatype.aether:aether-api:jar:1.12:compile
- org.sonatype.aether:aether-connector-file:jar:1.12:compile
- org.sonatype.aether:aether-connector-wagon:jar:1.12:compile
- org.sonatype.aether:aether-impl:jar:1.12:compile
- org.sonatype.aether:aether-spi:jar:1.12:compile
- org.sonatype.aether:aether-util:jar:1.12:compile
- org.sonatype.sisu:sisu-guice:jar:no_aop:3.0.2:compile
- org.sonatype.sisu:sisu-inject-bean:jar:2.2.2:compile
- org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
+org.codehaus.plexus:plexus-classworlds:jar:2.4:compile
+org.codehaus.plexus:plexus-component-annotations:jar:1.5.5:compile
+org.codehaus.plexus:plexus-interpolation:jar:1.14:compile
+org.codehaus.plexus:plexus-utils:jar:2.0.7:compile
+org.jsoup:jsoup:jar:1.6.1:compile
+org.sonatype.aether:aether-api:jar:1.12:compile
+org.sonatype.aether:aether-connector-file:jar:1.12:compile
+org.sonatype.aether:aether-connector-wagon:jar:1.12:compile
+org.sonatype.aether:aether-impl:jar:1.12:compile
+org.sonatype.aether:aether-spi:jar:1.12:compile
+org.sonatype.aether:aether-util:jar:1.12:compile
+org.sonatype.sisu:sisu-guice:jar:no_aop:3.0.2:compile
+org.sonatype.sisu:sisu-inject-bean:jar:2.2.2:compile
+org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
com.zaxxer:HikariCP:jar:2.7.8:compile,ASLv2,https://github.com/brettwooldridge/HikariCP
org.hibernate.validator:hibernate-validator:jar:6.0.9.Final:compile,ASLv2,https://github.com/hibernate/hibernate-validator
com.github.palindromicity:simple-syslog-5424:jar:0.0.8:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
[19/51] [abbrv] metron git commit: METRON-1794 Include User Details
When Escalating Alerts (nickwallen) closes apache/metron#1212
Posted by ot...@apache.org.
METRON-1794 Include User Details When Escalating Alerts (nickwallen) closes apache/metron#1212
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c0fb2625
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c0fb2625
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c0fb2625
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: c0fb26258f9a78d375a50dd5d746404130dfe06b
Parents: 9c9e295
Author: nickwallen <ni...@nickallen.org>
Authored: Mon Oct 1 09:49:17 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Mon Oct 1 09:49:17 2018 -0400
----------------------------------------------------------------------
.../apache/metron/rest/MetronRestConstants.java | 3 +
.../rest/service/impl/AlertsUIServiceImpl.java | 53 ++++++++++++----
.../service/impl/AlertsUIServiceImplTest.java | 63 ++++++++++++++++----
.../src/test/resources/log4j.properties | 5 +-
4 files changed, 98 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/c0fb2625/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
index e3bf698..94e8e35 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
@@ -53,6 +53,9 @@ public class MetronRestConstants {
public static final String KAFKA_BROKER_URL_SPRING_PROPERTY = "kafka.broker.url";
public static final String KAFKA_TOPICS_ESCALATION_PROPERTY = "kafka.topics.escalation";
+ public static final String METRON_ESCALATION_USER_FIELD = "metron_escalation_user";
+ public static final String METRON_ESCALATION_TIMESTAMP_FIELD = "metron_escalation_timestamp";
+
public static final String KERBEROS_ENABLED_SPRING_PROPERTY = "kerberos.enabled";
public static final String KERBEROS_PRINCIPLE_SPRING_PROPERTY = "kerberos.principal";
public static final String KERBEROS_KEYTAB_SPRING_PROPERTY = "kerberos.keytab";
http://git-wip-us.apache.org/repos/asf/metron/blob/c0fb2625/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
index 7d0a8f8..bf035e2 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
@@ -19,26 +19,32 @@ package org.apache.metron.rest.service.impl;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.metron.common.system.Clock;
import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.rest.MetronRestConstants;
+import org.apache.metron.hbase.client.UserSettingsClient;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.AlertsUIUserSettings;
-import org.apache.metron.hbase.client.UserSettingsClient;
import org.apache.metron.rest.security.SecurityUtils;
import org.apache.metron.rest.service.AlertsUIService;
import org.apache.metron.rest.service.KafkaService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.apache.metron.rest.MetronRestConstants.KAFKA_TOPICS_ESCALATION_PROPERTY;
+import static org.apache.metron.rest.MetronRestConstants.METRON_ESCALATION_TIMESTAMP_FIELD;
+import static org.apache.metron.rest.MetronRestConstants.METRON_ESCALATION_USER_FIELD;
+
/**
* The default service layer implementation of {@link AlertsUIService}.
*
@@ -47,6 +53,7 @@ import org.springframework.stereotype.Service;
@Service
public class AlertsUIServiceImpl implements AlertsUIService {
+ static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String ALERT_USER_SETTING_TYPE = "metron-alerts-ui";
public static ThreadLocal<ObjectMapper> _mapper = ThreadLocal.withInitial(() ->
new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL));
@@ -54,6 +61,7 @@ public class AlertsUIServiceImpl implements AlertsUIService {
private Environment environment;
private final KafkaService kafkaService;
private UserSettingsClient userSettingsClient;
+ private Clock clock;
@Autowired
public AlertsUIServiceImpl(final KafkaService kafkaService,
@@ -62,15 +70,25 @@ public class AlertsUIServiceImpl implements AlertsUIService {
this.kafkaService = kafkaService;
this.environment = environment;
this.userSettingsClient = userSettingsClient;
+ this.clock = new Clock();
}
@Override
public void escalateAlerts(List<Map<String, Object>> alerts) throws RestException {
+ String user = SecurityUtils.getCurrentUser();
+ String topic = environment.getProperty(KAFKA_TOPICS_ESCALATION_PROPERTY);
+ Long now = clock.currentTimeMillis();
+ LOG.info("Escalating {} alert(s): user={}, topic={}, timestamp={}", alerts.size(), user, topic, now);
+
try {
for (Map<String, Object> alert : alerts) {
- kafkaService.produceMessage(
- environment.getProperty(MetronRestConstants.KAFKA_TOPICS_ESCALATION_PROPERTY),
- JSONUtils.INSTANCE.toJSON(alert, false));
+ // attribute the escalation to the current user
+ alert.put(METRON_ESCALATION_USER_FIELD, user);
+ alert.put(METRON_ESCALATION_TIMESTAMP_FIELD, now);
+
+ // serialize the alert and push it to the escalation topic
+ String message = JSONUtils.INSTANCE.toJSON(alert, false);
+ kafkaService.produceMessage(topic, message);
}
} catch (JsonProcessingException e) {
throw new RestException(e);
@@ -128,4 +146,15 @@ public class AlertsUIServiceImpl implements AlertsUIService {
}
return success;
}
+
+ /**
+ * Set the {@link Clock} used by this service.
+ *
+ * <p>Calling this method is only needed to override the default behavior. This is useful when testing.
+ *
+ * @param clock
+ */
+ public void setClock(Clock clock) {
+ this.clock = clock;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/c0fb2625/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/AlertsUIServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/AlertsUIServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/AlertsUIServiceImplTest.java
index dc52712..545d7f9 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/AlertsUIServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/AlertsUIServiceImplTest.java
@@ -41,6 +41,7 @@ import java.util.Optional;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.system.FakeClock;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.model.AlertsUIUserSettings;
import org.apache.metron.hbase.client.UserSettingsClient;
@@ -79,9 +80,10 @@ public class AlertsUIServiceImplTest {
private KafkaService kafkaService;
private Environment environment;
private UserSettingsClient userSettingsClient;
- private AlertsUIService alertsUIService;
+ private AlertsUIServiceImpl alertsUIService;
private String user1 = "user1";
private String user2 = "user2";
+ private FakeClock clock;
@SuppressWarnings("unchecked")
@Before
@@ -91,6 +93,11 @@ public class AlertsUIServiceImplTest {
userSettingsClient = mock(UserSettingsClient.class);
alertsUIService = new AlertsUIServiceImpl(kafkaService, environment, userSettingsClient);
+ // use a fake clock for testing
+ clock = new FakeClock();
+ clock.elapseSeconds(1000);
+ alertsUIService.setClock(clock);
+
// assume user1 is logged in for tests
Authentication authentication = Mockito.mock(Authentication.class);
UserDetails userDetails = Mockito.mock(UserDetails.class);
@@ -100,21 +107,26 @@ public class AlertsUIServiceImplTest {
}
@Test
- public void produceMessageShouldProperlyProduceMessage() throws Exception {
- String escalationTopic = "escalation";
- final Map<String, Object> message1 = new HashMap<>();
- message1.put("field", "value1");
- final Map<String, Object> message2 = new HashMap<>();
- message2.put("field", "value2");
- List<Map<String, Object>> messages = Arrays.asList(message1, message2);
+ public void escalateAlertShouldSendMessageToKafka() throws Exception {
+ final String field = "field";
+ final String value1 = "value1";
+ final String value2 = "value2";
+
+ // define the escalation topic
+ final String escalationTopic = "escalation";
when(environment.getProperty(MetronRestConstants.KAFKA_TOPICS_ESCALATION_PROPERTY)).thenReturn(escalationTopic);
- alertsUIService.escalateAlerts(messages);
+ // create an alert along with the expected escalation message that is sent to kafka
+ final Map<String, Object> alert1 = mapOf(field, value1);
+ String escalationMessage1 = escalationMessage(field, value1, user1, clock.currentTimeMillis());
+
+ final Map<String, Object> alert2 = mapOf(field, value2);
+ String escalationMessage2 = escalationMessage(field, value2, user1, clock.currentTimeMillis());
- String expectedMessage1 = "{\"field\":\"value1\"}";
- String expectedMessage2 = "{\"field\":\"value2\"}";
- verify(kafkaService).produceMessage("escalation", expectedMessage1);
- verify(kafkaService).produceMessage("escalation", expectedMessage2);
+ // escalate the alerts and validate
+ alertsUIService.escalateAlerts(Arrays.asList(alert1, alert2));
+ verify(kafkaService).produceMessage(escalationTopic, escalationMessage1);
+ verify(kafkaService).produceMessage(escalationTopic, escalationMessage2);
verifyZeroInteractions(kafkaService);
}
@@ -177,4 +189,29 @@ public class AlertsUIServiceImplTest {
verify(userSettingsClient, times(2)).delete(user1, AlertsUIServiceImpl.ALERT_USER_SETTING_TYPE);
verifyNoMoreInteractions(userSettingsClient);
}
+
+ /**
+ * Defines what the message sent to Kafka should look-like when an alert is escalated.
+ *
+ * @param field The field name.
+ * @param value The value of the field.
+ * @param user The user who escalated the alert.
+ * @param timestamp When the alert was escalated.
+ * @return The escalated message.
+ */
+ private String escalationMessage(String field, String value, String user, Long timestamp) {
+ return String.format("{\"%s\":\"%s\",\"%s\":\"%s\",\"%s\":%d}",
+ field,
+ value,
+ MetronRestConstants.METRON_ESCALATION_USER_FIELD,
+ user,
+ MetronRestConstants.METRON_ESCALATION_TIMESTAMP_FIELD,
+ timestamp);
+ }
+
+ private Map<String, Object> mapOf(String key, Object value) {
+ Map<String, Object> map = new HashMap<>();
+ map.put(key, value);
+ return map;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/c0fb2625/metron-interface/metron-rest/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/resources/log4j.properties b/metron-interface/metron-rest/src/test/resources/log4j.properties
index 492cecf..edf866e 100644
--- a/metron-interface/metron-rest/src/test/resources/log4j.properties
+++ b/metron-interface/metron-rest/src/test/resources/log4j.properties
@@ -13,4 +13,7 @@
log4j.rootLogger=ERROR, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd'T'HH:mm:ss.SSS} %-5p [%c] - %m%n
\ No newline at end of file
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd'T'HH:mm:ss.SSS} %-5p [%c] - %m%n
+
+# uncomment the following line to enable debug
+#log4j.logger.org.apache.metron.rest=DEBUG
[18/51] [abbrv] metron git commit: METRON-1782 Add Kafka Partition
and Offset to Profiler Debug Logs (nickwallen) closes apache/metron#1202
Posted by ot...@apache.org.
METRON-1782 Add Kafka Partition and Offset to Profiler Debug Logs (nickwallen) closes apache/metron#1202
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9c9e2954
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9c9e2954
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9c9e2954
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 9c9e29547837e648723920329c3b7fea6211f0db
Parents: 1723a0e
Author: nickwallen <ni...@nickallen.org>
Authored: Mon Oct 1 09:07:28 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Mon Oct 1 09:07:28 2018 -0400
----------------------------------------------------------------------
.../src/main/flux/profiler/remote.yaml | 11 +++++++++--
.../metron/profiler/storm/ProfileSplitterBolt.java | 13 ++++++++++++-
.../metron/profiler/storm/ProfileSplitterBoltTest.java | 3 ++-
.../storm/kafka/flux/SimpleStormKafkaBuilder.java | 4 +++-
4 files changed, 26 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/9c9e2954/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml b/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml
index da71b27..e16a782 100644
--- a/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml
+++ b/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml
@@ -77,8 +77,15 @@ components:
className: "java.util.ArrayList"
configMethods:
- name: "add"
- args:
- - "value"
+ args: ["value"]
+ - name: "add"
+ args: ["topic"]
+ - name: "add"
+ args: ["partition"]
+ - name: "add"
+ args: ["offset"]
+ - name: "add"
+ args: ["timestamp"]
- id: "kafkaConfig"
className: "org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder"
http://git-wip-us.apache.org/repos/asf/metron/blob/9c9e2954/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
index 81179b6..ef58ad9 100644
--- a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
+++ b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
@@ -43,6 +43,12 @@ import java.lang.invoke.MethodHandles;
import java.util.List;
import java.util.Map;
+import static org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.OFFSET;
+import static org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.PARTITION;
+import static org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.TIMESTAMP;
+import static org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.TOPIC;
+import static org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.VALUE;
+
/**
* The Storm bolt responsible for filtering incoming messages and directing
* each to the downstream bolts responsible for building a Profile.
@@ -132,6 +138,11 @@ public class ProfileSplitterBolt extends ConfiguredProfilerBolt {
@Override
public void execute(Tuple input) {
try {
+ LOG.debug("Received message; topic={}, partition={}, offset={}, kafkaTimestamp={}",
+ input.contains(TOPIC.getFieldName()) ? input.getStringByField(TOPIC.getFieldName()): "unknown",
+ input.contains(PARTITION.getFieldName()) ? input.getIntegerByField(PARTITION.getFieldName()): "unknown",
+ input.contains(OFFSET.getFieldName()) ? input.getLongByField(OFFSET.getFieldName()): "unknown",
+ input.contains(TIMESTAMP.getFieldName()) ? input.getLongByField(TIMESTAMP.getFieldName()): "unknown");
doExecute(input);
} catch (Throwable t) {
@@ -146,7 +157,7 @@ public class ProfileSplitterBolt extends ConfiguredProfilerBolt {
private void doExecute(Tuple input) throws ParseException, UnsupportedEncodingException {
// retrieve the input message
- byte[] data = input.getBinary(0);
+ byte[] data = input.getBinaryByField(VALUE.getFieldName());
if(data == null) {
LOG.debug("Received null message. Nothing to do.");
return;
http://git-wip-us.apache.org/repos/asf/metron/blob/9c9e2954/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java
index 93d2ac4..360ef4b 100644
--- a/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java
+++ b/metron-analytics/metron-profiler-storm/src/test/java/org/apache/metron/profiler/storm/ProfileSplitterBoltTest.java
@@ -39,6 +39,7 @@ import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.HashMap;
+import static org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.VALUE;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.times;
@@ -219,7 +220,7 @@ public class ProfileSplitterBoltTest extends BaseBoltTest {
message = (JSONObject) parser.parse(input);
// ensure the tuple returns the expected json message
- when(tuple.getBinary(0)).thenReturn(input.getBytes());
+ when(tuple.getBinaryByField(VALUE.getFieldName())).thenReturn(input.getBytes());
}
/**
http://git-wip-us.apache.org/repos/asf/metron/blob/9c9e2954/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SimpleStormKafkaBuilder.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SimpleStormKafkaBuilder.java b/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SimpleStormKafkaBuilder.java
index f99e549..0aebff8 100644
--- a/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SimpleStormKafkaBuilder.java
+++ b/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SimpleStormKafkaBuilder.java
@@ -57,7 +57,9 @@ public class SimpleStormKafkaBuilder<K, V> extends KafkaSpoutConfig.Builder<K, V
KEY("key", record -> record.key()),
VALUE("value", record -> record.value()),
PARTITION("partition", record -> record.partition()),
- TOPIC("topic", record -> record.topic())
+ OFFSET("offset", record -> record.offset()),
+ TOPIC("topic", record -> record.topic()),
+ TIMESTAMP("timestamp", record -> record.timestamp())
;
String fieldName;
Function<ConsumerRecord,Object> recordExtractor;
[41/51] [abbrv] metron git commit: METRON-1805 Provide a default
value for the Storm topology.max.spout.pending setting (merrimanr) closes
apache/metron#1221
Posted by ot...@apache.org.
METRON-1805 Provide a default value for the Storm topology.max.spout.pending setting (merrimanr) closes apache/metron#1221
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/62141506
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/62141506
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/62141506
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 621415067dc727838da8e14925c15120b0e479de
Parents: 6478966
Author: merrimanr <me...@gmail.com>
Authored: Mon Oct 15 14:50:10 2018 -0500
Committer: rmerriman <me...@gmail.com>
Committed: Mon Oct 15 14:50:10 2018 -0500
----------------------------------------------------------------------
.../METRON/CURRENT/configuration/metron-indexing-env.xml | 4 ++--
.../METRON/CURRENT/configuration/metron-profiler-env.xml | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/62141506/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
index 3d268b7..ebc9759 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
@@ -146,7 +146,7 @@
<property>
<name>ra_indexing_topology_max_spout_pending</name>
<description>Indexing Topology Spout Max Pending Tuples</description>
- <value/>
+ <value>300</value>
<display-name>Indexing Max Pending for Random Access</display-name>
<value-attributes>
<empty-value-valid>true</empty-value-valid>
@@ -155,7 +155,7 @@
<property>
<name>batch_indexing_topology_max_spout_pending</name>
<description>Indexing Topology Spout Max Pending Tuples</description>
- <value/>
+ <value>300</value>
<display-name>Indexing Max Pending for HDFS</display-name>
<value-attributes>
<empty-value-valid>true</empty-value-valid>
http://git-wip-us.apache.org/repos/asf/metron/blob/62141506/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml
index 07b8d11..c45d799 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml
@@ -165,7 +165,7 @@
<name>profiler_topology_max_spout_pending</name>
<description>Profiler Topology Spout Max Pending Tuples</description>
<display-name>Spout Max Pending Tuples</display-name>
- <value/>
+ <value>300</value>
<value-attributes>
<empty-value-valid>true</empty-value-valid>
</value-attributes>
[32/51] [abbrv] metron git commit: METRON-1809 Support Column
Oriented Input with Batch Profiler (nickwallen) closes apache/metron#1229
Posted by ot...@apache.org.
METRON-1809 Support Column Oriented Input with Batch Profiler (nickwallen) closes apache/metron#1229
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3467ffdb
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3467ffdb
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3467ffdb
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 3467ffdb13aad22ab1e4065a0b6715b99371aa23
Parents: e482366
Author: nickwallen <ni...@nickallen.org>
Authored: Tue Oct 9 13:55:45 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Tue Oct 9 13:55:45 2018 -0400
----------------------------------------------------------------------
.../metron-profiler-spark/README.md | 53 +++++++--
.../metron/profiler/spark/BatchProfiler.java | 21 ++--
.../profiler/spark/BatchProfilerConfig.java | 9 +-
.../reader/ColumnEncodedTelemetryReader.java | 84 +++++++++++++
.../profiler/spark/reader/TelemetryReader.java | 43 +++++++
.../profiler/spark/reader/TelemetryReaders.java | 110 +++++++++++++++++
.../reader/TextEncodedTelemetryReader.java | 83 +++++++++++++
.../spark/BatchProfilerIntegrationTest.java | 36 +++++-
.../ColumnEncodedTelemetryReaderTest.java | 118 +++++++++++++++++++
.../spark/function/reader/IsValidJSON.java | 38 ++++++
.../function/reader/TelemetryReadersTest.java | 89 ++++++++++++++
.../reader/TextEncodedTelemetryReaderTest.java | 114 ++++++++++++++++++
12 files changed, 770 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/3467ffdb/metron-analytics/metron-profiler-spark/README.md
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/README.md b/metron-analytics/metron-profiler-spark/README.md
index df143f1..5ed5f4b 100644
--- a/metron-analytics/metron-profiler-spark/README.md
+++ b/metron-analytics/metron-profiler-spark/README.md
@@ -162,13 +162,13 @@ ${SPARK_HOME}/bin/spark-submit \
The Batch Profiler accepts the following arguments when run from the command line as shown above. All arguments following the Profiler jar are passed to the Profiler. All argument preceeding the Profiler jar are passed to Spark.
-| Argument | Description
-|--- |---
-| -p, --profiles | Path to the profile definitions.
-| -c, --config | Path to the profiler properties file.
-| -g, --globals | Path to the Stellar global config file.
-| -r, --reader | Path to properties for the DataFrameReader.
-| -h, --help | Print the help text.
+| Argument | Description
+|--- |---
+| [`-p`, `--profiles`](#--profiles) | Path to the profile definitions.
+| [`-c`, `--config`](#--config) | Path to the profiler properties file.
+| [`-g`, `--globals`](#--globals) | Path to the Stellar global config file.
+| [`-r`, `--reader`](#--reader) | Path to properties for the DataFrameReader.
+| `-h`, `--help` | Print the help text.
#### `--profiles`
@@ -234,6 +234,28 @@ The Profiler can consume archived telemetry stored in a variety of input formats
1. If additional options are required for your input format, then use the [`--reader`](#--reader) command-line argument when launching the Batch Profiler as [described here](#advanced-usage).
+#### Common Formats
+
+The following examples highlight the configuration values needed to read telemetry stored in common formats. These values should be defined in the Profiler properties (see [`--config`](#--config)).
+
+##### JSON
+```
+profiler.batch.input.reader=json
+profiler.batch.input.path=/path/to/json/
+```
+
+##### [Apache ORC](https://orc.apache.org/)
+```
+profiler.batch.input.reader=orc
+profiler.batch.input.path=/path/to/orc/
+```
+
+##### [Apache Parquet](http://parquet.apache.org/)
+```
+profiler.batch.input.reader=parquet
+profiler.batch.input.path=/path/to/parquet/
+```
+
## Configuring the Profiler
@@ -244,6 +266,7 @@ You can store both settings for the Profiler along with settings for Spark in th
| Setting | Description
|--- |---
| [`profiler.batch.input.path`](#profilerbatchinputpath) | The path to the input data read by the Batch Profiler.
+| [`profiler.batch.input.reader`](#profilerbatchinputreader) | The telemetry reader used to read the input data.
| [`profiler.batch.input.format`](#profilerbatchinputformat) | The format of the input data read by the Batch Profiler.
| [`profiler.batch.input.begin`](#profilerbatchinputend) | Only messages with a timestamp after this will be profiled.
| [`profiler.batch.input.end`](#profilerbatchinputbegin) | Only messages with a timestamp before this will be profiled.
@@ -259,11 +282,25 @@ You can store both settings for the Profiler along with settings for Spark in th
The path to the input data read by the Batch Profiler.
+### `profiler.batch.input.reader`
+
+*Default*: json
+
+Defines how the input data is treated when read. The value is not case sensitive so `JSON` and `json` are equivalent.
+
+ * `json`: Read text/json formatted telemetry
+ * `orc`: Read [Apache ORC](https://orc.apache.org/) formatted telemetry
+ * `parquet`: Read [Apache Parquet](http://parquet.apache.org/) formatted telemetry
+ * `text` Consumes input data stored as raw text. Should be defined along with [`profiler.batch.input.format`](#profilerbatchinputformat). Only use if the input format is not directly supported like `json`.
+ * `columnar` Consumes input data stored in columnar formats. Should be defined along with [`profiler.batch.input.format`](#profilerbatchinputformat). Only use if the input format is not directly supported like `json`.
+
+See [Common Formats](#common-formats) for further information.
+
### `profiler.batch.input.format`
*Default*: text
-The format of the input data read by the Batch Profiler.
+The format of the input data read by the Batch Profiler. This is optional and not required in most cases. For example, this property is not required when [`profiler.batch.input.reader`](#profilerbatchinputreader) is `json`, `orc`, or `parquet`.
### `profiler.batch.input.begin`
http://git-wip-us.apache.org/repos/asf/metron/blob/3467ffdb/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
index 39f8b3a..571545e 100644
--- a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
@@ -26,6 +26,8 @@ import org.apache.metron.profiler.spark.function.GroupByPeriodFunction;
import org.apache.metron.profiler.spark.function.HBaseWriterFunction;
import org.apache.metron.profiler.spark.function.MessageRouterFunction;
import org.apache.metron.profiler.spark.function.ProfileBuilderFunction;
+import org.apache.metron.profiler.spark.reader.TelemetryReader;
+import org.apache.metron.profiler.spark.reader.TelemetryReaders;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.SparkSession;
@@ -40,8 +42,7 @@ import java.util.Properties;
import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_BEGIN;
import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_END;
-import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
-import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_READER;
import static org.apache.spark.sql.functions.sum;
/**
@@ -54,6 +55,7 @@ public class BatchProfiler implements Serializable {
protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private TimestampParser timestampParser;
+ private TelemetryReader reader;
public BatchProfiler() {
this.timestampParser = new TimestampParser();
@@ -77,17 +79,10 @@ public class BatchProfiler implements Serializable {
LOG.debug("Building {} profile(s)", profiles.getProfiles().size());
Map<String, String> globals = Maps.fromProperties(globalProperties);
- String inputFormat = TELEMETRY_INPUT_FORMAT.get(profilerProps, String.class);
- String inputPath = TELEMETRY_INPUT_PATH.get(profilerProps, String.class);
- LOG.debug("Loading telemetry from '{}'", inputPath);
-
- // fetch the archived telemetry
- Dataset<String> telemetry = spark
- .read()
- .options(Maps.fromProperties(readerProps))
- .format(inputFormat)
- .load(inputPath)
- .as(Encoders.STRING());
+
+ // fetch the archived telemetry using the input reader
+ TelemetryReader reader = TelemetryReaders.create(TELEMETRY_INPUT_READER.get(profilerProps, String.class));
+ Dataset<String> telemetry = reader.read(spark, profilerProps, readerProps);
LOG.debug("Found {} telemetry record(s)", telemetry.cache().count());
// find all routes for each message
http://git-wip-us.apache.org/repos/asf/metron/blob/3467ffdb/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
index e8cd160..148d970 100644
--- a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
@@ -25,6 +25,9 @@ import org.apache.metron.stellar.common.utils.ConversionUtils;
import java.util.Map;
import java.util.Properties;
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.JSON;
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.TEXT;
+
/**
* Defines the configuration values recognized by the Batch Profiler.
*/
@@ -44,9 +47,11 @@ public enum BatchProfilerConfig {
HBASE_WRITE_DURABILITY("profiler.hbase.durability", Durability.USE_DEFAULT, Durability.class),
- TELEMETRY_INPUT_FORMAT("profiler.batch.input.format", "text", String.class),
+ TELEMETRY_INPUT_READER("profiler.batch.input.reader", JSON.toString(), String.class),
+
+ TELEMETRY_INPUT_FORMAT("profiler.batch.input.format", "", String.class),
- TELEMETRY_INPUT_PATH("profiler.batch.input.path", "hdfs://localhost:9000/apps/metron/indexing/indexed/*/*", String.class),
+ TELEMETRY_INPUT_PATH("profiler.batch.input.path", "hdfs://localhost:8020/apps/metron/indexing/indexed/*/*", String.class),
TELEMETRY_INPUT_BEGIN("profiler.batch.input.begin", "", String.class),
http://git-wip-us.apache.org/repos/asf/metron/blob/3467ffdb/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/ColumnEncodedTelemetryReader.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/ColumnEncodedTelemetryReader.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/ColumnEncodedTelemetryReader.java
new file mode 100644
index 0000000..41bf9b1
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/ColumnEncodedTelemetryReader.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.spark.reader;
+
+import com.google.common.collect.Maps;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Properties;
+
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+
+/**
+ * Reads in a {@link Dataset} then converts all of the {@link Dataset}'s column
+ * into a single JSON-formatted string.
+ *
+ * <p>This {@link TelemetryReader} is useful for any column-oriented format that
+ * is supported by Spark. For example, ORC and Parquet.
+ */
+public class ColumnEncodedTelemetryReader implements TelemetryReader {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * The input format to use when reading telemetry.
+ */
+ private String inputFormat;
+
+ /**
+ * Creates a {@link ColumnEncodedTelemetryReader}.
+ *
+ * <p>The input format used to read the telemetry is defined by the
+ * BatchProfilerConfig.TELEMETRY_INPUT_PATH property.
+ */
+ public ColumnEncodedTelemetryReader() {
+ this.inputFormat = null;
+ }
+
+ /**
+ * Creates a {@link ColumnEncodedTelemetryReader}.
+ *
+ * @param inputFormat The input format to use when reading telemetry.
+ */
+ public ColumnEncodedTelemetryReader(String inputFormat) {
+ this.inputFormat = inputFormat;
+ }
+
+ @Override
+ public Dataset<String> read(SparkSession spark, Properties profilerProps, Properties readerProps) {
+ String inputPath = TELEMETRY_INPUT_PATH.get(profilerProps, String.class);
+ if(inputFormat == null) {
+ inputFormat = TELEMETRY_INPUT_FORMAT.get(profilerProps, String.class);
+ }
+ LOG.debug("Loading telemetry; inputPath={}, inputFormat={}", inputPath, inputFormat);
+
+ return spark
+ .read()
+ .options(Maps.fromProperties(readerProps))
+ .format(inputFormat)
+ .load(inputPath)
+ .toJSON();
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3467ffdb/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReader.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReader.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReader.java
new file mode 100644
index 0000000..18861c0
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReader.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.spark.reader;
+
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.SparkSession;
+
+import java.io.Serializable;
+import java.util.Properties;
+
+/**
+ * A {@link TelemetryReader} is responsible for creating a {@link Dataset} containing
+ * telemetry that can be consumed by the {@link org.apache.metron.profiler.spark.BatchProfiler}.
+ */
+public interface TelemetryReader extends Serializable {
+
+ /**
+ * Read in the telemetry
+ *
+ * @param spark The spark session.
+ * @param profilerProps The profiler properties.
+ * @param readerProps The properties specific to reading input data.
+ * @return A {@link Dataset} containing archived telemetry.
+ */
+ Dataset<String> read(SparkSession spark, Properties profilerProps, Properties readerProps);
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3467ffdb/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReaders.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReaders.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReaders.java
new file mode 100644
index 0000000..0646155
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TelemetryReaders.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.spark.reader;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Properties;
+import java.util.function.Supplier;
+
+/**
+ * Allows a user to easily define the value of the property
+ * {@link org.apache.metron.profiler.spark.BatchProfilerConfig#TELEMETRY_INPUT_READER}.
+ */
+public enum TelemetryReaders implements TelemetryReader {
+
+ /**
+ * A {@link TelemetryReader} that is able to consume text/json formatted data.
+ *
+ * <p>This serves as a configuration short-cut for users. The user only needs to define the
+ * {@link org.apache.metron.profiler.spark.BatchProfilerConfig#TELEMETRY_INPUT_READER} property
+ * with this value to consume text/json.
+ */
+ JSON(() -> new TextEncodedTelemetryReader("text")),
+
+ /**
+ * A {@link TelemetryReader} that is able to consume Apache ORC formatted data.
+ *
+ * <p>This serves as a configuration short-cut for users. The user only needs to define the
+ * {@link org.apache.metron.profiler.spark.BatchProfilerConfig#TELEMETRY_INPUT_READER} property
+ * with this value to consume Apache ORC formatted data.
+ */
+ ORC(() -> new ColumnEncodedTelemetryReader("org.apache.spark.sql.execution.datasources.orc")),
+
+ /**
+ * A {@link TelemetryReader} that is able to consume Apache Parquet formatted data.
+ *
+ * <p>This serves as a configuration short-cut for users. The user only needs to define the
+ * {@link org.apache.metron.profiler.spark.BatchProfilerConfig#TELEMETRY_INPUT_READER} property
+ * with this value to consume Apache Parquet formatted data.
+ */
+ PARQUET(() -> new ColumnEncodedTelemetryReader("parquet")),
+
+ /**
+ * Use a {@link TextEncodedTelemetryReader} by defining the property value as 'TEXT'.
+ */
+ TEXT(() -> new TextEncodedTelemetryReader()),
+
+ /**
+ * Use a {@link ColumnEncodedTelemetryReader} by defining the property value as 'COLUMNAR'.
+ */
+ COLUMNAR(() -> new ColumnEncodedTelemetryReader());
+
+ static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private Supplier<TelemetryReader> supplier;
+
+ private TelemetryReaders(Supplier<TelemetryReader> supplier) {
+ this.supplier = supplier;
+ }
+
+ /**
+ * Returns a {@link TelemetryReader} based on a property value.
+ *
+ * @param propertyValue The property value.
+ * @return A {@link TelemetryReader}
+ * @throws IllegalArgumentException If the property value is invalid.
+ */
+ public static TelemetryReader create(String propertyValue) {
+ LOG.debug("Creating telemetry reader: telemetryReader={}", propertyValue);
+ TelemetryReader reader = null;
+ try {
+ String key = StringUtils.upperCase(propertyValue);
+ TelemetryReaders strategy = TelemetryReaders.valueOf(key);
+ reader = strategy.supplier.get();
+
+ } catch(IllegalArgumentException e) {
+ LOG.error("Unexpected telemetry reader: telemetryReader=" + propertyValue, e);
+ throw e;
+ }
+
+ return reader;
+ }
+
+ @Override
+ public Dataset<String> read(SparkSession spark, Properties profilerProps, Properties readerProps) {
+ return supplier.get().read(spark, profilerProps, readerProps);
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3467ffdb/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TextEncodedTelemetryReader.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TextEncodedTelemetryReader.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TextEncodedTelemetryReader.java
new file mode 100644
index 0000000..1f10b97
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/reader/TextEncodedTelemetryReader.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.spark.reader;
+
+import com.google.common.collect.Maps;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Properties;
+
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+
+/**
+ * A {@link TelemetryReader} that consumes telemetry stored as raw text.
+ *
+ * <p>This {@link TelemetryReader} is useful for any text-encoded formats like JSON and CSV.
+ */
+public class TextEncodedTelemetryReader implements TelemetryReader {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * The input format to use when reading telemetry.
+ */
+ private String inputFormat;
+
+ /**
+ * Creates a {@link TextEncodedTelemetryReader}.
+ *
+ * <p>The input format used to read the telemetry is defined by the
+ * BatchProfilerConfig.TELEMETRY_INPUT_PATH property.
+ */
+ public TextEncodedTelemetryReader() {
+ this.inputFormat = null;
+ }
+
+ /**
+ * Creates a {@link TextEncodedTelemetryReader}.
+ *
+ * @param inputFormat The input format to use when reading telemetry.
+ */
+ public TextEncodedTelemetryReader(String inputFormat) {
+ this.inputFormat = inputFormat;
+ }
+
+ @Override
+ public Dataset<String> read(SparkSession spark, Properties profilerProps, Properties readerProps) {
+ String inputPath = TELEMETRY_INPUT_PATH.get(profilerProps, String.class);
+ if(inputFormat == null) {
+ inputFormat = TELEMETRY_INPUT_FORMAT.get(profilerProps, String.class);
+ }
+ LOG.debug("Loading telemetry; inputPath={}, inputFormat={}", inputPath, inputFormat);
+
+ return spark
+ .read()
+ .options(Maps.fromProperties(readerProps))
+ .format(inputFormat)
+ .load(inputPath)
+ .as(Encoders.STRING());
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3467ffdb/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
index c33644f..83800af 100644
--- a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
@@ -58,8 +58,11 @@ import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INP
import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_END;
import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_READER;
import static org.junit.Assert.assertTrue;
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.*;
+
/**
* An integration test for the {@link BatchProfiler}.
*/
@@ -159,8 +162,8 @@ public class BatchProfilerIntegrationTest {
@Test
public void testBatchProfilerWithJSON() throws Exception {
// the input telemetry is text/json stored in the local filesystem
+ profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), JSON.toString());
profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), "src/test/resources/telemetry.json");
- profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "text");
BatchProfiler profiler = new BatchProfiler();
profiler.run(spark, profilerProperties, getGlobals(), readerProperties, getProfile());
@@ -170,20 +173,41 @@ public class BatchProfilerIntegrationTest {
@Test
public void testBatchProfilerWithORC() throws Exception {
- // re-write the test data as ORC
+ // re-write the test data as column-oriented ORC
String pathToORC = tempFolder.getRoot().getAbsolutePath();
spark.read()
- .format("text")
+ .format("json")
.load("src/test/resources/telemetry.json")
- .as(Encoders.STRING())
.write()
.mode("overwrite")
.format("org.apache.spark.sql.execution.datasources.orc")
.save(pathToORC);
// tell the profiler to use the ORC input data
+ profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), ORC.toString());
profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToORC);
- profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "org.apache.spark.sql.execution.datasources.orc");
+
+ BatchProfiler profiler = new BatchProfiler();
+ profiler.run(spark, profilerProperties, getGlobals(), readerProperties, getProfile());
+
+ validateProfiles();
+ }
+
+ @Test
+ public void testBatchProfilerWithParquet() throws Exception {
+ // re-write the test data as column-oriented ORC
+ String inputPath = tempFolder.getRoot().getAbsolutePath();
+ spark.read()
+ .format("json")
+ .load("src/test/resources/telemetry.json")
+ .write()
+ .mode("overwrite")
+ .format("parquet")
+ .save(inputPath);
+
+ // tell the profiler to use the ORC input data
+ profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), PARQUET.toString());
+ profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), inputPath);
BatchProfiler profiler = new BatchProfiler();
profiler.run(spark, profilerProperties, getGlobals(), readerProperties, getProfile());
@@ -206,7 +230,9 @@ public class BatchProfilerIntegrationTest {
.save(pathToCSV);
// tell the profiler to use the CSV input data
+ // CSV is an example of needing to define both the reader and the input format
profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToCSV);
+ profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), "text");
profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "csv");
// set a reader property; tell the reader to expect a header
http://git-wip-us.apache.org/repos/asf/metron/blob/3467ffdb/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/ColumnEncodedTelemetryReaderTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/ColumnEncodedTelemetryReaderTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/ColumnEncodedTelemetryReaderTest.java
new file mode 100644
index 0000000..0bcfb3f
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/ColumnEncodedTelemetryReaderTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function.reader;
+
+import org.apache.metron.profiler.spark.reader.TelemetryReaders;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.SparkSession;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.util.Properties;
+
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+
+/**
+ * Tests the {@link org.apache.metron.profiler.spark.reader.ColumnEncodedTelemetryReader} class.
+ */
+public class ColumnEncodedTelemetryReaderTest {
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+ private static SparkSession spark;
+ private Properties profilerProperties;
+ private Properties readerProperties;
+
+ @BeforeClass
+ public static void setupSpark() {
+ SparkConf conf = new SparkConf()
+ .setMaster("local")
+ .setAppName("BatchProfilerIntegrationTest")
+ .set("spark.sql.shuffle.partitions", "8");
+ spark = SparkSession
+ .builder()
+ .config(conf)
+ .getOrCreate();
+ }
+
+ @AfterClass
+ public static void tearDownSpark() {
+ if(spark != null) {
+ spark.close();
+ }
+ }
+
+ @Before
+ public void setup() {
+ readerProperties = new Properties();
+ profilerProperties = new Properties();
+ }
+
+ @Test
+ public void testParquet() {
+ // re-write the test data as column-oriented ORC
+ String inputPath = tempFolder.getRoot().getAbsolutePath();
+ spark.read()
+ .format("json")
+ .load("src/test/resources/telemetry.json")
+ .write()
+ .mode("overwrite")
+ .format("parquet")
+ .save(inputPath);
+
+ // tell the profiler to use the CSV input data
+ profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), inputPath);
+ profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "parquet");
+
+ // set a reader property; tell the reader to expect a header
+ readerProperties.put("header", "true");
+
+ // there should be 100 valid JSON records
+ Dataset<String> telemetry = TelemetryReaders.COLUMNAR.read(spark, profilerProperties, readerProperties);
+ Assert.assertEquals(100, telemetry.filter(new IsValidJSON()).count());
+ }
+
+ @Test
+ public void testORC() {
+ // re-write the test data as column-oriented ORC
+ String pathToORC = tempFolder.getRoot().getAbsolutePath();
+ spark.read()
+ .format("json")
+ .load("src/test/resources/telemetry.json")
+ .write()
+ .mode("overwrite")
+ .format("org.apache.spark.sql.execution.datasources.orc")
+ .save(pathToORC);
+
+ // tell the profiler to use the CSV input data
+ profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToORC);
+ profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "org.apache.spark.sql.execution.datasources.orc");
+
+ // there should be 100 valid JSON records
+ Dataset<String> telemetry = TelemetryReaders.COLUMNAR.read(spark, profilerProperties, readerProperties);
+ Assert.assertEquals(100, telemetry.filter(new IsValidJSON()).count());
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3467ffdb/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/IsValidJSON.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/IsValidJSON.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/IsValidJSON.java
new file mode 100644
index 0000000..50144f9
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/IsValidJSON.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function.reader;
+
+import org.apache.spark.api.java.function.FilterFunction;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+
+/**
+ * A filter function that filters out invalid JSON records.
+ */
+public class IsValidJSON implements FilterFunction<String> {
+
+ @Override
+ public boolean call(String text) throws Exception {
+ JSONParser parser = new JSONParser();
+ JSONObject json = (JSONObject) parser.parse(text);
+
+ // all of the test data has at least 32 fields in each JSON record
+ return json.keySet().size() >= 32;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3467ffdb/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TelemetryReadersTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TelemetryReadersTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TelemetryReadersTest.java
new file mode 100644
index 0000000..e525ae0
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TelemetryReadersTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function.reader;
+
+import org.apache.metron.profiler.spark.reader.ColumnEncodedTelemetryReader;
+import org.apache.metron.profiler.spark.reader.TelemetryReaders;
+import org.apache.metron.profiler.spark.reader.TextEncodedTelemetryReader;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.COLUMNAR;
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.JSON;
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.ORC;
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.PARQUET;
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.TEXT;
+
+public class TelemetryReadersTest {
+
+ @Test
+ public void testJsonReader() {
+ String key = JSON.toString();
+ Assert.assertTrue(TelemetryReaders.create(key) instanceof TextEncodedTelemetryReader);
+ }
+
+ @Test
+ public void testJsonReaderLowerCase() {
+ String key = JSON.toString().toLowerCase();
+ Assert.assertTrue(TelemetryReaders.create(key) instanceof TextEncodedTelemetryReader);
+ }
+
+ @Test
+ public void testOrcReader() {
+ String key = ORC.toString();
+ Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader);
+ }
+
+
+ @Test
+ public void testOrcReaderLowerCase() {
+ String key = ORC.toString().toLowerCase();
+ Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader);
+ }
+
+ @Test
+ public void testParquetReader() {
+ String key = PARQUET.toString();
+ Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader);
+ }
+
+ @Test
+ public void testParquetReaderLowerCase() {
+ String key = PARQUET.toString().toLowerCase();
+ Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader);
+ }
+
+ @Test
+ public void testTextReader() {
+ String key = TEXT.toString();
+ Assert.assertTrue(TelemetryReaders.create(key) instanceof TextEncodedTelemetryReader);
+ }
+
+ @Test
+ public void testColumnReader() {
+ String key = COLUMNAR.toString();
+ Assert.assertTrue(TelemetryReaders.create(key) instanceof ColumnEncodedTelemetryReader);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidReader() {
+ TelemetryReaders.create("invalid");
+ Assert.fail("exception expected");
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3467ffdb/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TextEncodedTelemetryReaderTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TextEncodedTelemetryReaderTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TextEncodedTelemetryReaderTest.java
new file mode 100644
index 0000000..3b26bb9
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/TextEncodedTelemetryReaderTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function.reader;
+
+import org.apache.metron.profiler.spark.reader.TelemetryReaders;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SparkSession;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.util.Properties;
+
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+
+/**
+ * Tests the {@link org.apache.metron.profiler.spark.reader.TextEncodedTelemetryReader} class.
+ */
+public class TextEncodedTelemetryReaderTest {
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+ private static SparkSession spark;
+ private Properties profilerProperties;
+ private Properties readerProperties;
+
+ @BeforeClass
+ public static void setupSpark() {
+ SparkConf conf = new SparkConf()
+ .setMaster("local")
+ .setAppName("BatchProfilerIntegrationTest")
+ .set("spark.sql.shuffle.partitions", "8");
+ spark = SparkSession
+ .builder()
+ .config(conf)
+ .getOrCreate();
+ }
+
+ @AfterClass
+ public static void tearDownSpark() {
+ if(spark != null) {
+ spark.close();
+ }
+ }
+
+ @Before
+ public void setup() {
+ readerProperties = new Properties();
+ profilerProperties = new Properties();
+ }
+
+ @Test
+ public void testCSV() {
+ // re-write the test data as a CSV with a header record
+ String pathToCSV = tempFolder.getRoot().getAbsolutePath();
+ spark.read()
+ .format("text")
+ .load("src/test/resources/telemetry.json")
+ .as(Encoders.STRING())
+ .write()
+ .mode("overwrite")
+ .option("header", "true")
+ .format("csv")
+ .save(pathToCSV);
+
+ // tell the profiler to use the CSV input data
+ profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToCSV);
+ profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "csv");
+
+ // set a reader property; tell the reader to expect a header
+ readerProperties.put("header", "true");
+
+ // there should be 100 valid JSON records
+ Dataset<String> telemetry = TelemetryReaders.TEXT.read(spark, profilerProperties, readerProperties);
+ Assert.assertEquals(100, telemetry.filter(new IsValidJSON()).count());
+ }
+
+ @Test
+ public void testJSON() {
+ // use the test telemetry that is stored as raw json
+ profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), "src/test/resources/telemetry.json");
+ profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "text");
+
+ // set a reader property; tell the reader to expect a header
+ readerProperties.put("header", "true");
+
+ // there should be 100 valid JSON records
+ Dataset<String> telemetry = TelemetryReaders.TEXT.read(spark, profilerProperties, readerProperties);
+ Assert.assertEquals(100, telemetry.filter(new IsValidJSON()).count());
+ }
+}
[22/51] [abbrv] metron git commit: METRON-1804 Update version to
0.6.1 (justinleet) closes apache/metron#1220
Posted by ot...@apache.org.
METRON-1804 Update version to 0.6.1 (justinleet) closes apache/metron#1220
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b872fdcd
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b872fdcd
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b872fdcd
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: b872fdcd0a54f41165e93d0e3635635a09e61ef8
Parents: ebdaf5f
Author: justinleet <ju...@gmail.com>
Authored: Wed Oct 3 15:25:44 2018 -0400
Committer: leet <le...@apache.org>
Committed: Wed Oct 3 15:25:44 2018 -0400
----------------------------------------------------------------------
metron-analytics/metron-maas-common/pom.xml | 2 +-
metron-analytics/metron-maas-service/pom.xml | 2 +-
metron-analytics/metron-profiler-client/pom.xml | 2 +-
metron-analytics/metron-profiler-common/pom.xml | 2 +-
metron-analytics/metron-profiler-repl/pom.xml | 2 +-
metron-analytics/metron-profiler-spark/pom.xml | 2 +-
metron-analytics/metron-profiler-storm/pom.xml | 2 +-
metron-analytics/metron-statistics/pom.xml | 2 +-
metron-analytics/pom.xml | 2 +-
metron-contrib/metron-docker/pom.xml | 2 +-
metron-contrib/metron-performance/pom.xml | 2 +-
metron-contrib/pom.xml | 2 +-
metron-deployment/amazon-ec2/conf/defaults.yml | 2 +-
metron-deployment/ansible/playbooks/docker_probe_install.yml | 2 +-
.../ansible/roles/ambari_master/defaults/main.yml | 2 +-
.../development/centos6/ansible/inventory/group_vars/all | 2 +-
.../development/ubuntu14/ansible/inventory/group_vars/all | 2 +-
metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml | 4 ++--
metron-deployment/packaging/ambari/metron-mpack/pom.xml | 4 ++--
.../ambari/metron-mpack/src/main/resources/mpack.json | 2 +-
metron-deployment/packaging/docker/deb-docker/pom.xml | 4 ++--
metron-deployment/packaging/docker/rpm-docker/pom.xml | 4 ++--
metron-deployment/pom.xml | 2 +-
metron-interface/metron-alerts/package-lock.json | 2 +-
metron-interface/metron-alerts/package.json | 2 +-
metron-interface/metron-alerts/pom.xml | 2 +-
metron-interface/metron-config/package-lock.json | 2 +-
metron-interface/metron-config/package.json | 2 +-
metron-interface/metron-config/pom.xml | 2 +-
metron-interface/metron-config/scripts/package.json | 2 +-
metron-interface/metron-rest-client/pom.xml | 2 +-
metron-interface/metron-rest/pom.xml | 2 +-
.../metron-rest/src/main/resources/application.yml | 2 +-
metron-interface/pom.xml | 2 +-
metron-platform/Performance-tuning-guide.md | 4 ++--
metron-platform/README.md | 2 +-
metron-platform/elasticsearch-shaded/pom.xml | 2 +-
metron-platform/metron-common/pom.xml | 2 +-
metron-platform/metron-common/src/main/scripts/cluster_info.py | 2 +-
metron-platform/metron-data-management/pom.xml | 2 +-
metron-platform/metron-elasticsearch/pom.xml | 2 +-
metron-platform/metron-enrichment/pom.xml | 2 +-
metron-platform/metron-hbase-client/pom.xml | 2 +-
metron-platform/metron-hbase/pom.xml | 2 +-
metron-platform/metron-indexing/pom.xml | 2 +-
metron-platform/metron-integration-test/pom.xml | 2 +-
metron-platform/metron-job/pom.xml | 2 +-
metron-platform/metron-management/pom.xml | 2 +-
metron-platform/metron-parsers/pom.xml | 2 +-
metron-platform/metron-pcap-backend/pom.xml | 2 +-
.../metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh | 2 +-
metron-platform/metron-pcap/pom.xml | 2 +-
metron-platform/metron-solr/pom.xml | 2 +-
metron-platform/metron-storm-kafka-override/pom.xml | 2 +-
metron-platform/metron-storm-kafka/pom.xml | 2 +-
metron-platform/metron-test-utilities/pom.xml | 2 +-
metron-platform/metron-writer/pom.xml | 2 +-
metron-platform/metron-zookeeper/pom.xml | 2 +-
metron-platform/pom.xml | 2 +-
metron-stellar/pom.xml | 2 +-
metron-stellar/stellar-3rd-party-example/pom.xml | 2 +-
metron-stellar/stellar-common/README.md | 4 ++--
metron-stellar/stellar-common/pom.xml | 2 +-
metron-stellar/stellar-zeppelin/README.md | 6 +++---
metron-stellar/stellar-zeppelin/pom.xml | 2 +-
pom.xml | 2 +-
site-book/pom.xml | 2 +-
67 files changed, 75 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-analytics/metron-maas-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-common/pom.xml b/metron-analytics/metron-maas-common/pom.xml
index bc1aa0c..b5cf1c9 100644
--- a/metron-analytics/metron-maas-common/pom.xml
+++ b/metron-analytics/metron-maas-common/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-analytics</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-maas-common</artifactId>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-analytics/metron-maas-service/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml
index ae19d30..17b80f6 100644
--- a/metron-analytics/metron-maas-service/pom.xml
+++ b/metron-analytics/metron-maas-service/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-analytics</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-maas-service</artifactId>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-analytics/metron-profiler-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml
index ae04740..2957b0f 100644
--- a/metron-analytics/metron-profiler-client/pom.xml
+++ b/metron-analytics/metron-profiler-client/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-analytics</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-profiler-client</artifactId>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-analytics/metron-profiler-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/pom.xml b/metron-analytics/metron-profiler-common/pom.xml
index b3aafcd..630709e 100644
--- a/metron-analytics/metron-profiler-common/pom.xml
+++ b/metron-analytics/metron-profiler-common/pom.xml
@@ -18,7 +18,7 @@
<parent>
<artifactId>metron-analytics</artifactId>
<groupId>org.apache.metron</groupId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-profiler-common</artifactId>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-analytics/metron-profiler-repl/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-repl/pom.xml b/metron-analytics/metron-profiler-repl/pom.xml
index 0876f70..4586a5d 100644
--- a/metron-analytics/metron-profiler-repl/pom.xml
+++ b/metron-analytics/metron-profiler-repl/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>metron-analytics</artifactId>
<groupId>org.apache.metron</groupId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-profiler-repl</artifactId>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-analytics/metron-profiler-spark/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/pom.xml b/metron-analytics/metron-profiler-spark/pom.xml
index 668ee2c..60ea30d 100644
--- a/metron-analytics/metron-profiler-spark/pom.xml
+++ b/metron-analytics/metron-profiler-spark/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-analytics</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-profiler-spark</artifactId>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-analytics/metron-profiler-storm/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-storm/pom.xml b/metron-analytics/metron-profiler-storm/pom.xml
index dd80467..cfa30b3 100644
--- a/metron-analytics/metron-profiler-storm/pom.xml
+++ b/metron-analytics/metron-profiler-storm/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-analytics</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-profiler-storm</artifactId>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-analytics/metron-statistics/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-statistics/pom.xml b/metron-analytics/metron-statistics/pom.xml
index 4119f55..732f247 100644
--- a/metron-analytics/metron-statistics/pom.xml
+++ b/metron-analytics/metron-statistics/pom.xml
@@ -18,7 +18,7 @@
<parent>
<artifactId>metron-analytics</artifactId>
<groupId>org.apache.metron</groupId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-statistics</artifactId>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-analytics/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/pom.xml b/metron-analytics/pom.xml
index d2d1263..bff6ba0 100644
--- a/metron-analytics/pom.xml
+++ b/metron-analytics/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<description>Stream analytics for Metron</description>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-contrib/metron-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-contrib/metron-docker/pom.xml b/metron-contrib/metron-docker/pom.xml
index d789605..15c7a5b 100644
--- a/metron-contrib/metron-docker/pom.xml
+++ b/metron-contrib/metron-docker/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-contrib</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<description>Metron Docker</description>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-contrib/metron-performance/pom.xml
----------------------------------------------------------------------
diff --git a/metron-contrib/metron-performance/pom.xml b/metron-contrib/metron-performance/pom.xml
index 55cb0a7..5f031a6 100644
--- a/metron-contrib/metron-performance/pom.xml
+++ b/metron-contrib/metron-performance/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-contrib</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<description>Performance Testing Utilities</description>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/metron-contrib/pom.xml b/metron-contrib/pom.xml
index 7233f3f..3c29cf6 100644
--- a/metron-contrib/pom.xml
+++ b/metron-contrib/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<description>Contributed Tooling for Metron</description>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-deployment/amazon-ec2/conf/defaults.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/conf/defaults.yml b/metron-deployment/amazon-ec2/conf/defaults.yml
index b4dc2e1..cd0e0a1 100644
--- a/metron-deployment/amazon-ec2/conf/defaults.yml
+++ b/metron-deployment/amazon-ec2/conf/defaults.yml
@@ -58,7 +58,7 @@ num_partitions: 3
retention_in_gb: 25
# metron variables
-metron_version: 0.6.0
+metron_version: 0.6.1
metron_directory: /usr/metron/{{ metron_version }}
# sensors
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-deployment/ansible/playbooks/docker_probe_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/playbooks/docker_probe_install.yml b/metron-deployment/ansible/playbooks/docker_probe_install.yml
index 563487d..824aa40 100644
--- a/metron-deployment/ansible/playbooks/docker_probe_install.yml
+++ b/metron-deployment/ansible/playbooks/docker_probe_install.yml
@@ -30,7 +30,7 @@
- hosts: sensors
vars:
- metron_version: 0.6.0
+ metron_version: 0.6.1
metron_directory: /usr/metron/{{ metron_version }}
bro_version: "2.5.4"
fixbuf_version: "1.7.1"
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
index 182ac53..8bae8ce 100644
--- a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
+++ b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
@@ -16,7 +16,7 @@
#
---
ambari_server_mem: 2048
-ambari_mpack_version: 0.6.0.0
+ambari_mpack_version: 0.6.1.0
metron_mpack_name: metron_mpack-{{ ambari_mpack_version }}.tar.gz
metron_mpack_path: "{{ playbook_dir }}/../../packaging/ambari/metron-mpack/target/{{ metron_mpack_name }}"
elasticsearch_mpack_name: elasticsearch_mpack-{{ ambari_mpack_version }}.tar.gz
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-deployment/development/centos6/ansible/inventory/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/ansible/inventory/group_vars/all b/metron-deployment/development/centos6/ansible/inventory/group_vars/all
index 550eb8f..694ecb7 100644
--- a/metron-deployment/development/centos6/ansible/inventory/group_vars/all
+++ b/metron-deployment/development/centos6/ansible/inventory/group_vars/all
@@ -35,7 +35,7 @@ threatintel_hbase_table: threatintel
enrichment_hbase_table: enrichment
# metron
-metron_version: 0.6.0
+metron_version: 0.6.1
metron_directory: /usr/metron/{{ metron_version }}
bro_version: "2.5.4"
fixbuf_version: "1.7.1"
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
index 1338d7c..f26e7f8 100644
--- a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
+++ b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
@@ -35,7 +35,7 @@ threatintel_hbase_table: threatintel
enrichment_hbase_table: enrichment
# metron
-metron_version: 0.6.0
+metron_version: 0.6.1
metron_directory: /usr/metron/{{ metron_version }}
bro_version: "2.5.4"
fixbuf_version: "1.7.1"
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml
index 7b9a21e..edd2d43 100644
--- a/metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.metron.packaging.mpacks</groupId>
<artifactId>elasticsearch_mpack</artifactId>
- <version>0.6.0.0</version>
+ <version>0.6.1.0</version>
<name>Elasticsearch Ambari Management Pack</name>
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-deployment</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
<relativePath>../../..</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-deployment/packaging/ambari/metron-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
index 557898b..82386c1 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.metron.packaging.mpacks</groupId>
<artifactId>metron_mpack</artifactId>
- <version>0.6.0.0</version>
+ <version>0.6.1.0</version>
<name>Metron Ambari Management Pack</name>
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-deployment</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
<relativePath>../../..</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
index e624dd0..bc67d16 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
@@ -1,7 +1,7 @@
{
"type": "full-release",
"name": "metron-ambari.mpack",
- "version": "0.6.0.0",
+ "version": "0.6.1.0",
"description": "Ambari Management Pack for Apache Metron",
"prerequisites": {
"min-ambari-version": "2.4.0.0",
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-deployment/packaging/docker/deb-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/pom.xml b/metron-deployment/packaging/docker/deb-docker/pom.xml
index e15c6dd..d691cd2 100644
--- a/metron-deployment/packaging/docker/deb-docker/pom.xml
+++ b/metron-deployment/packaging/docker/deb-docker/pom.xml
@@ -21,11 +21,11 @@
<artifactId>metron-debs</artifactId>
<packaging>pom</packaging>
<name>metron-debs</name>
- <version>0.6.0</version>
+ <version>0.6.1</version>
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-deployment</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
<relativePath>../../..</relativePath>
</parent>
<description>DEB Package Builder for Apache Metron</description>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-deployment/packaging/docker/rpm-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index d8f032f..fc8ca4b 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -21,11 +21,11 @@
<artifactId>metron-rpm</artifactId>
<packaging>pom</packaging>
<name>metron-rpm</name>
- <version>0.6.0</version>
+ <version>0.6.1</version>
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-deployment</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
<relativePath>../../..</relativePath>
</parent>
<description>RPM Builder for Apache Metron</description>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-deployment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/pom.xml b/metron-deployment/pom.xml
index da94ec5..ee92447 100644
--- a/metron-deployment/pom.xml
+++ b/metron-deployment/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<description>Building and deploying Metron</description>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-interface/metron-alerts/package-lock.json
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/package-lock.json b/metron-interface/metron-alerts/package-lock.json
index f927ffe..bb2512e 100644
--- a/metron-interface/metron-alerts/package-lock.json
+++ b/metron-interface/metron-alerts/package-lock.json
@@ -1,6 +1,6 @@
{
"name": "metron-alerts",
- "version": "0.6.0",
+ "version": "0.6.1",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-interface/metron-alerts/package.json
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/package.json b/metron-interface/metron-alerts/package.json
index 14172fd..ef72bb7 100644
--- a/metron-interface/metron-alerts/package.json
+++ b/metron-interface/metron-alerts/package.json
@@ -1,6 +1,6 @@
{
"name": "metron-alerts",
- "version": "0.6.0",
+ "version": "0.6.1",
"license": "Apache-2.0",
"angular-cli": {},
"scripts": {
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-interface/metron-alerts/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/pom.xml b/metron-interface/metron-alerts/pom.xml
index aafcb94..417df93 100644
--- a/metron-interface/metron-alerts/pom.xml
+++ b/metron-interface/metron-alerts/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-interface</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-alerts</artifactId>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-interface/metron-config/package-lock.json
----------------------------------------------------------------------
diff --git a/metron-interface/metron-config/package-lock.json b/metron-interface/metron-config/package-lock.json
index 26fbe4e..fff8cb7 100644
--- a/metron-interface/metron-config/package-lock.json
+++ b/metron-interface/metron-config/package-lock.json
@@ -1,6 +1,6 @@
{
"name": "metron-management-ui",
- "version": "0.6.0",
+ "version": "0.6.1",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-interface/metron-config/package.json
----------------------------------------------------------------------
diff --git a/metron-interface/metron-config/package.json b/metron-interface/metron-config/package.json
index 6a9d8ba..6e196d5 100644
--- a/metron-interface/metron-config/package.json
+++ b/metron-interface/metron-config/package.json
@@ -1,6 +1,6 @@
{
"name": "metron-management-ui",
- "version": "0.6.0",
+ "version": "0.6.1",
"license": "Apache-2.0",
"config": {
"node_ace": "node_modules/ace-builds/src-min-noconflict/"
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-interface/metron-config/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-config/pom.xml b/metron-interface/metron-config/pom.xml
index e30a2ea..fd76447 100644
--- a/metron-interface/metron-config/pom.xml
+++ b/metron-interface/metron-config/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-interface</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-config</artifactId>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-interface/metron-config/scripts/package.json
----------------------------------------------------------------------
diff --git a/metron-interface/metron-config/scripts/package.json b/metron-interface/metron-config/scripts/package.json
index 8ed7504..4d75d90 100644
--- a/metron-interface/metron-config/scripts/package.json
+++ b/metron-interface/metron-config/scripts/package.json
@@ -1,6 +1,6 @@
{
"name": "metron-management-ui-web-server",
- "version": "0.6.0",
+ "version": "0.6.1",
"description": "Metron management ui web server",
"main": "server.js",
"dependencies": {
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-interface/metron-rest-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/pom.xml b/metron-interface/metron-rest-client/pom.xml
index 09767cd..1781874 100644
--- a/metron-interface/metron-rest-client/pom.xml
+++ b/metron-interface/metron-rest-client/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-interface</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-rest-client</artifactId>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-interface/metron-rest/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml
index de73cf9..7fc373b 100644
--- a/metron-interface/metron-rest/pom.xml
+++ b/metron-interface/metron-rest/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-interface</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-rest</artifactId>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-interface/metron-rest/src/main/resources/application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml
index fd6d5a6..19ea186 100644
--- a/metron-interface/metron-rest/src/main/resources/application.yml
+++ b/metron-interface/metron-rest/src/main/resources/application.yml
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
metron:
- version: 0.6.0
+ version: 0.6.1
logging:
level:
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-interface/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/pom.xml b/metron-interface/pom.xml
index 3c4e28f..763d7c3 100644
--- a/metron-interface/pom.xml
+++ b/metron-interface/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<description>Interfaces for Metron</description>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/Performance-tuning-guide.md
----------------------------------------------------------------------
diff --git a/metron-platform/Performance-tuning-guide.md b/metron-platform/Performance-tuning-guide.md
index ab292a3..bfc36dd 100644
--- a/metron-platform/Performance-tuning-guide.md
+++ b/metron-platform/Performance-tuning-guide.md
@@ -410,7 +410,7 @@ And we ran our bro parser topology with the following options. We did not need t
though you could certainly do so if necessary. Notice that we only needed 1 worker.
```
-/usr/metron/0.6.0/bin/start_parser_topology.sh \
+/usr/metron/0.6.1/bin/start_parser_topology.sh \
-e ~metron/.storm/storm-bro.config \
-esc ~/.storm/spout-bro.config \
-k $BROKERLIST \
@@ -594,7 +594,7 @@ export KAFKA_HOME=$HDP_HOME/kafka-broker
export STORM_UI=http://node1:8744
export ELASTIC=http://node1:9200
export ZOOKEEPER=node1:2181
-export METRON_VERSION=0.6.0
+export METRON_VERSION=0.6.1
export METRON_HOME=/usr/metron/${METRON_VERSION}
```
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/README.md b/metron-platform/README.md
index fa9ff73..93424ea 100644
--- a/metron-platform/README.md
+++ b/metron-platform/README.md
@@ -17,7 +17,7 @@ limitations under the License.
# Current Build
-The latest build of metron-platform is 0.6.0.
+The latest build of metron-platform is 0.6.1.
We are still in the process of merging/porting additional features from our production code base into this open source release. This release will be followed by a number of additional beta releases until the port is complete. We will also work on getting additional documentation and user/developer guides to the community as soon as we can. At this time we offer no support for the beta software, but will try to respond to requests as promptly as we can.
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/elasticsearch-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml
index 6864a0d..ccad3cb 100644
--- a/metron-platform/elasticsearch-shaded/pom.xml
+++ b/metron-platform/elasticsearch-shaded/pom.xml
@@ -17,7 +17,7 @@
<parent>
<artifactId>metron-platform</artifactId>
<groupId>org.apache.metron</groupId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>elasticsearch-shaded</artifactId>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index f5acc5f..1ca8601 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-common</artifactId>
<name>metron-common</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-common/src/main/scripts/cluster_info.py
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/scripts/cluster_info.py b/metron-platform/metron-common/src/main/scripts/cluster_info.py
index da440ef..86643e4 100755
--- a/metron-platform/metron-common/src/main/scripts/cluster_info.py
+++ b/metron-platform/metron-common/src/main/scripts/cluster_info.py
@@ -338,7 +338,7 @@ class ClusterInfo:
type="string",
dest="metron_home",
help="Metron home directory",
- default="/usr/metron/0.6.0",
+ default="/usr/metron/0.6.1",
metavar="DIRECTORY")
parser.add_option("-p", "--hdp_home",
action="store",
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-data-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml
index 8f55932..c4dbb74 100644
--- a/metron-platform/metron-data-management/pom.xml
+++ b/metron-platform/metron-data-management/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-data-management</artifactId>
<name>metron-data-management</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index 9cc974d..adc601a 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-elasticsearch</artifactId>
<name>metron-elasticsearch</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-enrichment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml
index e50f23f..56ded42 100644
--- a/metron-platform/metron-enrichment/pom.xml
+++ b/metron-platform/metron-enrichment/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-enrichment</artifactId>
<name>metron-enrichment</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-hbase-client/pom.xml b/metron-platform/metron-hbase-client/pom.xml
index 147d637..185aff8 100644
--- a/metron-platform/metron-hbase-client/pom.xml
+++ b/metron-platform/metron-hbase-client/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<!--
The purpose of this project is to provide a version of the hbase client which has a shaded and relocated gauva.
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-hbase/pom.xml b/metron-platform/metron-hbase/pom.xml
index 6a00790..fe44bb9 100644
--- a/metron-platform/metron-hbase/pom.xml
+++ b/metron-platform/metron-hbase/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-hbase</artifactId>
<name>metron-hbase</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-indexing/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml
index 02a32e9..5be1714 100644
--- a/metron-platform/metron-indexing/pom.xml
+++ b/metron-platform/metron-indexing/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-indexing</artifactId>
<name>metron-indexing</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-integration-test/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml
index 2b801db..c3d1e5a 100644
--- a/metron-platform/metron-integration-test/pom.xml
+++ b/metron-platform/metron-integration-test/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-integration-test</artifactId>
<name>metron-integration-test</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-job/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-job/pom.xml b/metron-platform/metron-job/pom.xml
index fb4713c..7cee88e 100644
--- a/metron-platform/metron-job/pom.xml
+++ b/metron-platform/metron-job/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-job</artifactId>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml
index 6c5a143..79578ca 100644
--- a/metron-platform/metron-management/pom.xml
+++ b/metron-platform/metron-management/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-management</artifactId>
<name>metron-management</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-parsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index 5077791..e804f71 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-parsers</artifactId>
<name>metron-parsers</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-pcap-backend/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml
index 614f877..19b509c 100644
--- a/metron-platform/metron-pcap-backend/pom.xml
+++ b/metron-platform/metron-pcap-backend/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-pcap-backend</artifactId>
<name>metron-pcap-backend</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh
index da739d6..da0fe2c 100755
--- a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh
+++ b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh
@@ -17,7 +17,7 @@
# limitations under the License.
#
-METRON_VERSION=0.6.0
+METRON_VERSION=0.6.1
METRON_HOME=${METRON_HOME:-"/usr/metron/$METRON_VERSION"}
DATE_FORMAT=${DATE_FORMAT:-"yyyyMMdd"}
USER=$(whoami)
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-pcap/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap/pom.xml b/metron-platform/metron-pcap/pom.xml
index c34cfe1..cd08f91 100644
--- a/metron-platform/metron-pcap/pom.xml
+++ b/metron-platform/metron-pcap/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-pcap</artifactId>
<name>metron-pcap</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-solr/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml
index 4332776..31ce769 100644
--- a/metron-platform/metron-solr/pom.xml
+++ b/metron-platform/metron-solr/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-solr</artifactId>
<name>metron-solr</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-storm-kafka-override/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-storm-kafka-override/pom.xml b/metron-platform/metron-storm-kafka-override/pom.xml
index 2b81cb4..b12766f 100644
--- a/metron-platform/metron-storm-kafka-override/pom.xml
+++ b/metron-platform/metron-storm-kafka-override/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-storm-kafka-override</artifactId>
<name>metron-storm-kafka-override</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-storm-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-storm-kafka/pom.xml b/metron-platform/metron-storm-kafka/pom.xml
index 752e7f4..f8bffef 100644
--- a/metron-platform/metron-storm-kafka/pom.xml
+++ b/metron-platform/metron-storm-kafka/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-storm-kafka</artifactId>
<name>metron-storm-kafka</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-test-utilities/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-test-utilities/pom.xml b/metron-platform/metron-test-utilities/pom.xml
index c2ff58e..f258d2f 100644
--- a/metron-platform/metron-test-utilities/pom.xml
+++ b/metron-platform/metron-test-utilities/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-test-utilities</artifactId>
<name>metron-test-utilities</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-writer/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml
index 1e65b46..e845516 100644
--- a/metron-platform/metron-writer/pom.xml
+++ b/metron-platform/metron-writer/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-writer</artifactId>
<name>metron-writer</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/metron-zookeeper/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-zookeeper/pom.xml b/metron-platform/metron-zookeeper/pom.xml
index 6f65140..d65cd90 100644
--- a/metron-platform/metron-zookeeper/pom.xml
+++ b/metron-platform/metron-zookeeper/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>metron-platform</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>metron-zookeeper</artifactId>
<name>metron-zookeeper</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-platform/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/pom.xml b/metron-platform/pom.xml
index 3209eb6..379c0a5 100644
--- a/metron-platform/pom.xml
+++ b/metron-platform/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<description>Stream analytics for Metron</description>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-stellar/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/pom.xml b/metron-stellar/pom.xml
index eace3ac..356fbbb 100644
--- a/metron-stellar/pom.xml
+++ b/metron-stellar/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>Metron</artifactId>
<groupId>org.apache.metron</groupId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<description>DSL for stream analytics</description>
<url>https://metron.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-stellar/stellar-3rd-party-example/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-3rd-party-example/pom.xml b/metron-stellar/stellar-3rd-party-example/pom.xml
index 938be51..2c0afc1 100644
--- a/metron-stellar/stellar-3rd-party-example/pom.xml
+++ b/metron-stellar/stellar-3rd-party-example/pom.xml
@@ -18,7 +18,7 @@
<parent>
<artifactId>metron-stellar</artifactId>
<groupId>org.apache.metron</groupId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>stellar-3rd-party-example</artifactId>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-stellar/stellar-common/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md
index 768d34d..468f358 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -1477,7 +1477,7 @@ operating system.
```bash
-metron-stellar/stellar-common/target/stellar-common-0.6.0-stand-alone.tar.gz
+metron-stellar/stellar-common/target/stellar-common-0.6.1-stand-alone.tar.gz
```
When unpacked, the following structure will be created:
@@ -1487,7 +1487,7 @@ When unpacked, the following structure will be created:
├── bin
│  └── stellar
└── lib
- └── stellar-common-0.6.0-uber.jar
+ └── stellar-common-0.6.1-uber.jar
```
To run the Stellar Shell run the following from the directory you unpacked to:
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-stellar/stellar-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml
index d3cc1ea..42e649d 100644
--- a/metron-stellar/stellar-common/pom.xml
+++ b/metron-stellar/stellar-common/pom.xml
@@ -18,7 +18,7 @@
<parent>
<artifactId>metron-stellar</artifactId>
<groupId>org.apache.metron</groupId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<artifactId>stellar-common</artifactId>
<name>stellar-common</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-stellar/stellar-zeppelin/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-zeppelin/README.md b/metron-stellar/stellar-zeppelin/README.md
index dddb6b0..97090d7 100644
--- a/metron-stellar/stellar-zeppelin/README.md
+++ b/metron-stellar/stellar-zeppelin/README.md
@@ -56,10 +56,10 @@ To install the Stellar Interpreter in your Apache Zeppelin installation, follow
1. Use Zeppelin's installation utility to install the Stellar Interpreter.
- If Zeppelin was already installed, make sure that it is stopped before running this command. Update the version, '0.6.0' in the example below, to whatever is appropriate for your environment.
+ If Zeppelin was already installed, make sure that it is stopped before running this command. Update the version, '0.6.1' in the example below, to whatever is appropriate for your environment.
```
- bin/install-interpreter.sh --name stellar --artifact org.apache.metron:stellar-zeppelin:0.6.0
+ bin/install-interpreter.sh --name stellar --artifact org.apache.metron:stellar-zeppelin:0.6.1
```
1. Start Zeppelin.
@@ -106,7 +106,7 @@ Usage
1. Go back to the Stellar interpreter configuration and add another dependency as follows.
```
- org.apache.metron:metron-statistics:0.6.0
+ org.apache.metron:metron-statistics:0.6.1
```
1. Go back to your notebook and run `%functions` again. You will now see the additional functions defined within the `metron-statistics` project.
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/metron-stellar/stellar-zeppelin/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-zeppelin/pom.xml b/metron-stellar/stellar-zeppelin/pom.xml
index dc3327f..6cdaa83 100644
--- a/metron-stellar/stellar-zeppelin/pom.xml
+++ b/metron-stellar/stellar-zeppelin/pom.xml
@@ -18,7 +18,7 @@
<parent>
<artifactId>metron-stellar</artifactId>
<groupId>org.apache.metron</groupId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<name>stellar-zeppelin</name>
<description>Stellar Interpreter for Apache Zeppelin</description>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9bde04e..a98de72 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
<packaging>pom</packaging>
<name>Metron</name>
<description>Metron Top Level Project</description>
http://git-wip-us.apache.org/repos/asf/metron/blob/b872fdcd/site-book/pom.xml
----------------------------------------------------------------------
diff --git a/site-book/pom.xml b/site-book/pom.xml
index d809080..fad52a0 100644
--- a/site-book/pom.xml
+++ b/site-book/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.metron</groupId>
<artifactId>Metron</artifactId>
- <version>0.6.0</version>
+ <version>0.6.1</version>
</parent>
<description>User Documentation for Metron</description>
<url>https://metron.apache.org/</url>
[20/51] [abbrv] metron git commit: METRON-1750 Create Parser for
Syslog RFC 5424 Messages (ottobackwards) closes apache/metron#1175
Posted by ot...@apache.org.
METRON-1750 Create Parser for Syslog RFC 5424 Messages (ottobackwards) closes apache/metron#1175
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ff1f9cf5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ff1f9cf5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ff1f9cf5
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: ff1f9cf52b31ae866eb3de1cf8993af97af1790f
Parents: c0fb262
Author: ottobackwards <ot...@gmail.com>
Authored: Mon Oct 1 10:23:38 2018 -0400
Committer: otto <ot...@apache.org>
Committed: Mon Oct 1 10:23:38 2018 -0400
----------------------------------------------------------------------
dependencies_with_url.csv | 2 +-
.../docker/rpm-docker/SPECS/metron.spec | 3 +
.../data/syslog5424/parsed/Syslog5424Parsed | 3 +
.../sample/data/syslog5424/raw/Syslog5424Output | 3 +
metron-platform/metron-parsers/README.md | 5 +-
metron-platform/metron-parsers/pom.xml | 5 +
.../config/zookeeper/parsers/syslog5424.json | 7 +
.../metron/parsers/syslog/Syslog5424Parser.java | 102 +++++++++++++
.../Syslog5424ParserIntegrationTest.java | 37 +++++
.../parsers/syslog/Syslog5424ParserTest.java | 146 +++++++++++++++++++
pom.xml | 1 +
11 files changed, 311 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index ffd0fbc..53977f3 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -488,4 +488,4 @@ com.google.code.gson:gson:jar:2.8.2:compile,ASLv2,https://github.com/google/gson
org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
com.zaxxer:HikariCP:jar:2.7.8:compile,ASLv2,https://github.com/brettwooldridge/HikariCP
org.hibernate.validator:hibernate-validator:jar:6.0.9.Final:compile,ASLv2,https://github.com/hibernate/hibernate-validator
-
+com.github.palindromicity:simple-syslog-5424:jar:0.0.8:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 4bcef33..ed22a28 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -158,6 +158,7 @@ This package installs the Metron Parser files
%{metron_home}/config/zookeeper/parsers/jsonMapWrappedQuery.json
%{metron_home}/config/zookeeper/parsers/snort.json
%{metron_home}/config/zookeeper/parsers/squid.json
+%{metron_home}/config/zookeeper/parsers/syslog5424.json
%{metron_home}/config/zookeeper/parsers/websphere.json
%{metron_home}/config/zookeeper/parsers/yaf.json
%{metron_home}/config/zookeeper/parsers/asa.json
@@ -590,6 +591,8 @@ chkconfig --del metron-alerts-ui
%changelog
* Thu Aug 30 2018 Apache Metron <de...@metron.apache.org> - 0.6.1
- Update compiled css file name for Alerts UI
+* Fri Aug 24 2018 Apache Metron <de...@metron.apache.org> - 0.6.1
+- Add syslog5424 parser
* Tue Aug 21 2018 Apache Metron <de...@metron.apache.org> - 0.6.1
- Add Profiler for REPL
* Tue Aug 14 2018 Apache Metron <de...@metron.apache.org> - 0.5.1
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
new file mode 100644
index 0000000..e330204
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
@@ -0,0 +1,3 @@
+{"syslog.header.appName":"d0602076-b14a-4c55-852a-981e7afeed38","syslog.header.version":"1","syslog.header.hostName":"loggregator","original_string":"<14>1 2014-06-20T09:14:07+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed38 DEA - - Removing instance","syslog.header.facility":"1","syslog.header.msgId":"-","syslog.header.timestamp":"2014-06-20T09:14:07+00:00","syslog.message":"Removing instance","syslog.header.pri":"14","syslog.header.procId":"DEA","syslog.header.severity":"6","timestamp":"2014-06-20T09:14:07+00:00","guid":"this-is-random-uuid-will-be-36-chars","source.type":"syslog5424"}
+{"syslog.structuredData.exampleSDID@32480.iut":"4","syslog.structuredData.exampleSDID@32480.eventSource":"Other Application","syslog.header.hostName":"loggregator","syslog.header.facility":"1","syslog.structuredData.exampleSDID@32480.eventID":"2022","syslog.structuredData.exampleSDID@32473.eventSource":"Application","syslog.header.timestamp":"2014-06-20T09:14:08+00:00","syslog.message":"Removing instance","syslog.header.pri":"14","syslog.header.procId":"DEA","syslog.header.severity":"6","syslog.header.appName":"d0602076-b14a-4c55-852a-981e7afeed40","syslog.header.version":"1","syslog.structuredData.exampleSDID@32473.iut":"3","original_string":"<14>1 2014-06-20T09:14:08+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed40 DEA MSG-02 [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"] [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"] Removing instance","syslog.header.msgId":"MSG-02","syslog.structuredData.exampleSDID@32473.eventI
D":"1011","timestamp":"2014-06-20T09:14:08+00:00","guid":"this-is-random-uuid-will-be-36-chars","source.type":"syslog5424"}
+{"syslog.structuredData.exampleSDID@32480.iut":"4","syslog.structuredData.exampleSDID@32480.eventSource":"Other Application","syslog.structuredData.exampleSDID@32474.iut":"3","syslog.structuredData.exampleSDID@32474.eventID":"1011","syslog.header.hostName":"loggregator","syslog.header.facility":"1","syslog.structuredData.exampleSDID@32480.eventID":"2022","syslog.header.timestamp":"2014-06-20T09:14:09+00:00","syslog.message":"Removing instance","syslog.header.pri":"14","syslog.header.procId":"DEA","syslog.structuredData.exampleSDID@32474.eventSource":"Application","syslog.header.severity":"6","syslog.header.appName":"d0602076-b14a-4c55-852a-981e7afeed42","syslog.header.version":"1","original_string":"<14>1 2014-06-20T09:14:09+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed42 DEA MSG-03 [exampleSDID@32474 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"] [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"] Removing instance","syslog.header.msgId"
:"MSG-03","timestamp":"2014-06-20T09:14:09+00:00","guid":"this-is-random-uuid-will-be-36-chars","source.type":"syslog5424"}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/raw/Syslog5424Output
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/raw/Syslog5424Output b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/raw/Syslog5424Output
new file mode 100644
index 0000000..869797a
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/raw/Syslog5424Output
@@ -0,0 +1,3 @@
+<14>1 2014-06-20T09:14:07+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed38 DEA - - Removing instance
+<14>1 2014-06-20T09:14:08+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed40 DEA MSG-02 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"] [exampleSDID@32480 iut="4" eventSource="Other Application" eventID="2022"] Removing instance
+<14>1 2014-06-20T09:14:09+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed42 DEA MSG-03 [exampleSDID@32474 iut="3" eventSource="Application" eventID="1011"] [exampleSDID@32480 iut="4" eventSource="Other Application" eventID="2022"] Removing instance
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md
index fd6b470..381b0ee 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -24,7 +24,7 @@ Parsers are pluggable components which are used to transform raw data
enrichment and indexing.
There are two general types types of parsers:
-* A parser written in Java which conforms to the `MessageParser` interface. This kind of parser is optimized for speed and performance and is built for use with higher velocity topologies. These parsers are not easily modifiable and in order to make changes to them the entire topology need to be recompiled.
+* A parser written in Java which conforms to the `MessageParser` interface. This kind of parser is optimized for speed and performance and is built for use with higher velocity topologies. These parsers are not easily modifiable and in order to make changes to them the entire topology need to be recompiled.
* A general purpose parser. This type of parser is primarily designed for lower-velocity topologies or for quickly standing up a parser for a new telemetry before a permanent Java parser can be written for it. As of the time of this writing, we have:
* Grok parser: `org.apache.metron.parsers.GrokParser` with possible `parserConfig` entries of
* `grokPath` : The path in HDFS (or in the Jar) to the grok statement
@@ -504,12 +504,13 @@ Parser adapters are loaded dynamically in each Metron topology. They
are defined in the Parser Config (defined above) JSON file in Zookeeper.
### Java Parser Adapters
-Java parser adapters are indended for higher-velocity topologies and are not easily changed or extended. As the adoption of Metron continues we plan on extending our library of Java adapters to process more log formats. As of this moment the Java adapters included with Metron are:
+Java parser adapters are intended for higher-velocity topologies and are not easily changed or extended. As the adoption of Metron continues we plan on extending our library of Java adapters to process more log formats. As of this moment the Java adapters included with Metron are:
* org.apache.metron.parsers.ise.BasicIseParser : Parse ISE messages
* org.apache.metron.parsers.bro.BasicBroParser : Parse Bro messages
* org.apache.metron.parsers.sourcefire.BasicSourcefireParser : Parse Sourcefire messages
* org.apache.metron.parsers.lancope.BasicLancopeParser : Parse Lancope messages
+* org.apache.metron.parsers.syslog.Syslog5424Parser : Parse Syslog RFC 5424 messages
### Grok Parser Adapters
Grok parser adapters are designed primarly for someone who is not a Java coder for quickly standing up a parser adapter for lower velocity topologies. Grok relies on Regex for message parsing, which is much slower than purpose-built Java parsers, but is more extensible. Grok parsers are defined via a config file and the topplogy does not need to be recombiled in order to make changes to them. An example of a Grok perser is:
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index e586492..5077791 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -261,6 +261,11 @@
<artifactId>json-path</artifactId>
<version>2.3.0</version>
</dependency>
+ <dependency>
+ <groupId>com.github.palindromicity</groupId>
+ <artifactId>simple-syslog-5424</artifactId>
+ <version>${global_simple_syslog_version}</version>
+ </dependency>
</dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json
new file mode 100644
index 0000000..5f62692
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json
@@ -0,0 +1,7 @@
+{
+ "parserClassName":"org.apache.metron.parsers.syslog.Syslog5424Parser",
+ "sensorTopic":"syslog5424",
+ "parserConfig": {
+ "nilPolicy": "DASH"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
new file mode 100644
index 0000000..e3ad941
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.syslog;
+
+import com.github.palindromicity.syslog.AllowableDeviations;
+import com.github.palindromicity.syslog.NilPolicy;
+import com.github.palindromicity.syslog.SyslogParser;
+import com.github.palindromicity.syslog.SyslogParserBuilder;
+import com.github.palindromicity.syslog.dsl.SyslogFieldKeys;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+
+import java.io.BufferedReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * Parser for well structured RFC 5424 messages.
+ */
+public class Syslog5424Parser extends BasicParser {
+ public static final String NIL_POLICY_CONFIG = "nilPolicy";
+ private transient SyslogParser syslogParser;
+
+ @Override
+ public void configure(Map<String, Object> config) {
+ // Default to OMIT policy for nil fields
+ // this means they will not be in the returned field set
+ String nilPolicyStr = (String) config.getOrDefault(NIL_POLICY_CONFIG, NilPolicy.OMIT.name());
+ NilPolicy nilPolicy = NilPolicy.valueOf(nilPolicyStr);
+ syslogParser = new SyslogParserBuilder()
+ .withNilPolicy(nilPolicy)
+ .withDeviations(EnumSet.of(AllowableDeviations.PRIORITY,AllowableDeviations.VERSION))
+ .build();
+ }
+
+ @Override
+ public void init() {
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public List<JSONObject> parse(byte[] rawMessage) {
+ try {
+ if (rawMessage == null || rawMessage.length == 0) {
+ return null;
+ }
+
+ String originalString = new String(rawMessage);
+ List<JSONObject> returnList = new ArrayList<>();
+ try (Reader reader = new BufferedReader(new StringReader(originalString))) {
+ syslogParser.parseLines(reader, (m) -> {
+ JSONObject jsonObject = new JSONObject(m);
+ // be sure to put in the original string, and the timestamp.
+ // we wil just copy over the timestamp from the syslog
+ jsonObject.put("original_string", originalString);
+ setTimestamp(jsonObject);
+ returnList.add(jsonObject);
+ });
+
+ return returnList;
+ }
+ } catch (Exception e) {
+ String message = "Unable to parse " + new String(rawMessage) + ": " + e.getMessage();
+ LOG.error(message, e);
+ throw new IllegalStateException(message, e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void setTimestamp(JSONObject message) {
+ String timeStampString = (String) message.get(SyslogFieldKeys.HEADER_TIMESTAMP.getField());
+ if (!StringUtils.isBlank(timeStampString) && !timeStampString.equals("-")) {
+ message.put("timestamp", timeStampString);
+ } else {
+ message.put("timestamp", LocalDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java
new file mode 100644
index 0000000..c9c47ce
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.integration;
+
+import org.apache.metron.parsers.integration.validation.SampleDataValidation;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Syslog5424ParserIntegrationTest extends ParserIntegrationTest {
+ @Override
+ String getSensorType() {
+ return "syslog5424";
+ }
+
+ @Override
+ List<ParserValidation> getValidations() {
+ return new ArrayList<ParserValidation>() {{
+ add(new SampleDataValidation());
+ }};
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
new file mode 100644
index 0000000..0ef26ff
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.syslog;
+
+import com.github.palindromicity.syslog.NilPolicy;
+import com.github.palindromicity.syslog.dsl.SyslogFieldKeys;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+
+public class Syslog5424ParserTest {
+ private static final String SYSLOG_LINE_ALL = "<14>1 2014-06-20T09:14:07+00:00 loggregator"
+ + " d0602076-b14a-4c55-852a-981e7afeed38 DEA MSG-01"
+ + " [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]"
+ + " [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"] Removing instance";
+
+ private static final String SYSLOG_LINE_MISSING = "<14>1 2014-06-20T09:14:07+00:00 loggregator"
+ + " d0602076-b14a-4c55-852a-981e7afeed38 DEA -"
+ + " [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]"
+ + " [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"] Removing instance";
+
+ private static final String SYSLOG_LINE_MISSING_DATE = "<14>1 - loggregator"
+ + " d0602076-b14a-4c55-852a-981e7afeed38 DEA -"
+ + " [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]"
+ + " [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"] Removing instance";
+
+ private static final String expectedVersion = "1";
+ private static final String expectedMessage = "Removing instance";
+ private static final String expectedAppName = "d0602076-b14a-4c55-852a-981e7afeed38";
+ private static final String expectedHostName = "loggregator";
+ private static final String expectedPri = "14";
+ private static final String expectedFacility = "1";
+ private static final String expectedSeverity = "6";
+ private static final String expectedProcId = "DEA";
+ private static final String expectedTimestamp = "2014-06-20T09:14:07+00:00";
+ private static final String expectedMessageId = "MSG-01";
+
+ private static final String expectedIUT1 = "3";
+ private static final String expectedIUT2 = "4";
+ private static final String expectedEventSource1 = "Application";
+ private static final String expectedEventSource2 = "Other Application";
+ private static final String expectedEventID1 = "1011";
+ private static final String expectedEventID2 = "2022";
+
+ @Test
+ public void testHappyPath() {
+ test(null, SYSLOG_LINE_ALL, (message) -> Assert.assertEquals(expectedMessageId, message.get(SyslogFieldKeys.HEADER_MSGID.getField())));
+ }
+
+ @Test
+ public void testOmit() {
+ test(NilPolicy.OMIT, SYSLOG_LINE_MISSING, (message) -> Assert.assertFalse(message.containsKey(SyslogFieldKeys.HEADER_MSGID)));
+ }
+
+ @Test
+ public void testDash() {
+ test(NilPolicy.DASH, SYSLOG_LINE_MISSING, (message) -> Assert.assertEquals("-", message.get(SyslogFieldKeys.HEADER_MSGID.getField())));
+ }
+
+ @Test()
+ public void testNull() {
+ test(NilPolicy.NULL, SYSLOG_LINE_MISSING, (message) -> {
+ Assert.assertTrue(message.containsKey(SyslogFieldKeys.HEADER_MSGID.getField()));
+ Assert.assertNull(message.get(SyslogFieldKeys.HEADER_MSGID.getField()));
+ });
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testNotValid() {
+ test(null, "not valid", (message) -> Assert.assertTrue(false));
+ }
+
+ public void test(NilPolicy nilPolicy, String line, Consumer<JSONObject> msgIdChecker) {
+ Syslog5424Parser parser = new Syslog5424Parser();
+ Map<String, Object> config = new HashMap<>();
+ if (nilPolicy != null) {
+ config.put(Syslog5424Parser.NIL_POLICY_CONFIG, nilPolicy.name());
+ }
+ parser.configure(config);
+
+ List<JSONObject> output = parser.parse(line.getBytes());
+ }
+
+ @Test
+ public void testReadMultiLine() throws Exception {
+ Syslog5424Parser parser = new Syslog5424Parser();
+ Map<String, Object> config = new HashMap<>();
+ config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.DASH.name());
+ parser.configure(config);
+ StringBuilder builder = new StringBuilder();
+ builder
+ .append(SYSLOG_LINE_ALL)
+ .append("\n")
+ .append(SYSLOG_LINE_MISSING)
+ .append("\n")
+ .append(SYSLOG_LINE_ALL);
+ List<JSONObject> output = parser.parse(builder.toString().getBytes());
+ Assert.assertEquals(3,output.size());
+ }
+
+ @Test
+ public void testMissingTimestamp() {
+ Syslog5424Parser parser = new Syslog5424Parser();
+ Map<String, Object> config = new HashMap<>();
+ config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.DASH.name());
+ parser.configure(config);
+ List<JSONObject> output = parser.parse(SYSLOG_LINE_MISSING_DATE.getBytes());
+ String timeStampString = output.get(0).get("timestamp").toString();
+ DateTimeFormatter.ISO_DATE_TIME.parse(timeStampString);
+ config.clear();
+ config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.NULL.name());
+ parser.configure(config);
+ output = parser.parse(SYSLOG_LINE_MISSING_DATE.getBytes());
+ timeStampString = output.get(0).get("timestamp").toString();
+ DateTimeFormatter.ISO_DATE_TIME.parse(timeStampString);
+
+ config.clear();
+ config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.OMIT.name());
+ parser.configure(config);
+ output = parser.parse(SYSLOG_LINE_MISSING_DATE.getBytes());
+ timeStampString = output.get(0).get("timestamp").toString();
+ DateTimeFormatter.ISO_DATE_TIME.parse(timeStampString);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1c1fa3d..9bde04e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -137,6 +137,7 @@
<global_reflections_version>0.9.10</global_reflections_version>
<global_checkstyle_version>8.0</global_checkstyle_version>
<global_log4j_core_version>2.1</global_log4j_core_version>
+ <global_simple_syslog_version>0.0.8</global_simple_syslog_version>
<global_spark_version>2.3.1</global_spark_version>
</properties>
[16/51] [abbrv] metron git commit: METRON-1699 Create Batch Profiler
(nickwallen)
Posted by ot...@apache.org.
METRON-1699 Create Batch Profiler (nickwallen)
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fbabfcad
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fbabfcad
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fbabfcad
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: fbabfcad49581b531a7b7eb755a561a030a2bc06
Parents: 4f0b608 fa3be8d
Author: Nick Allen <ni...@nickallen.org>
Authored: Fri Sep 28 08:22:36 2018 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Fri Sep 28 08:22:36 2018 -0400
----------------------------------------------------------------------
README.md | 2 +-
dependencies_with_url.csv | 66 +-
.../client/stellar/ProfilerFunctions.java | 283 ------
.../client/stellar/ProfilerFunctionsTest.java | 334 -------
.../metron-profiler-common/README.md | 386 ++++++++
.../profiler/DefaultMessageDistributor.java | 33 +-
.../metron/profiler/DefaultMessageRouter.java | 37 +-
.../metron/profiler/DefaultProfileBuilder.java | 4 +-
.../metron/profiler/MessageDistributor.java | 4 +-
.../apache/metron/profiler/MessageRoute.java | 76 +-
.../metron/profiler/ProfileMeasurement.java | 2 +-
.../apache/metron/profiler/ProfilePeriod.java | 51 +-
.../metron/profiler/StandAloneProfiler.java | 186 ----
.../profiler/clock/DefaultClockFactory.java | 4 +-
.../metron/profiler/clock/EventTimeClock.java | 3 +-
.../clock/EventTimeOnlyClockFactory.java | 58 ++
.../profiler/DefaultMessageDistributorTest.java | 32 +-
.../profiler/DefaultMessageRouterTest.java | 55 ++
.../profiler/DefaultProfileBuilderTest.java | 6 +-
.../metron/profiler/ProfilePeriodTest.java | 51 +-
.../metron/profiler/StandAloneProfilerTest.java | 255 ------
.../clock/EventTimeOnlyClockFactoryTest.java | 61 ++
metron-analytics/metron-profiler-repl/README.md | 179 ++++
metron-analytics/metron-profiler-repl/pom.xml | 150 ++++
.../src/main/assembly/assembly.xml | 36 +
.../metron/profiler/repl/ProfilerFunctions.java | 278 ++++++
.../profiler/repl/StandAloneProfiler.java | 176 ++++
.../profiler/repl/ProfilerFunctionsTest.java | 332 +++++++
.../profiler/repl/StandAloneProfilerTest.java | 255 ++++++
.../metron-profiler-spark/README.md | 317 +++++++
metron-analytics/metron-profiler-spark/pom.xml | 222 +++++
.../src/main/assembly/assembly.xml | 60 ++
.../src/main/config/batch-profiler.properties | 26 +
.../metron/profiler/spark/BatchProfiler.java | 139 +++
.../profiler/spark/BatchProfilerConfig.java | 194 ++++
.../spark/ProfileMeasurementAdapter.java | 132 +++
.../metron/profiler/spark/TimestampParser.java | 55 ++
.../profiler/spark/cli/BatchProfilerCLI.java | 200 +++++
.../spark/cli/BatchProfilerCLIOptions.java | 150 ++++
.../spark/function/GroupByPeriodFunction.java | 60 ++
.../spark/function/HBaseWriterFunction.java | 171 ++++
.../spark/function/MessageRouterFunction.java | 203 +++++
.../spark/function/ProfileBuilderFunction.java | 107 +++
.../profiler/spark/function/TaskUtils.java | 41 +
.../src/main/scripts/start_batch_profiler.sh | 32 +
.../spark/BatchProfilerIntegrationTest.java | 324 +++++++
.../profiler/spark/TimestampParserTest.java | 67 ++
.../spark/cli/BatchProfilerCLITest.java | 61 ++
.../spark/function/HBaseWriterFunctionTest.java | 176 ++++
.../function/MessageRouterFunctionTest.java | 229 +++++
.../function/ProfileBuilderFunctionTest.java | 98 ++
.../src/test/resources/globals.json | 3 +
.../src/test/resources/log4j.properties | 31 +
.../src/test/resources/profiles-empty.json | 5 +
.../resources/profiles-no-timestamp-field.json | 12 +
.../src/test/resources/profiles.json | 13 +
.../src/test/resources/telemetry.json | 100 +++
.../metron-profiler-storm/.gitignore | 1 +
.../metron-profiler-storm/README.md | 400 +++++++++
metron-analytics/metron-profiler-storm/pom.xml | 407 +++++++++
.../src/main/assembly/assembly.xml | 72 ++
.../src/main/config/profiler.properties | 71 ++
.../src/main/flux/profiler/remote.yaml | 218 +++++
.../storm/FixedFrequencyFlushSignal.java | 135 +++
.../metron/profiler/storm/FlushSignal.java | 51 ++
.../metron/profiler/storm/HBaseEmitter.java | 73 ++
.../metron/profiler/storm/KafkaEmitter.java | 164 ++++
.../profiler/storm/ManualFlushSignal.java | 54 ++
.../profiler/storm/ProfileBuilderBolt.java | 552 ++++++++++++
.../profiler/storm/ProfileHBaseMapper.java | 117 +++
.../storm/ProfileMeasurementEmitter.java | 59 ++
.../profiler/storm/ProfileSplitterBolt.java | 228 +++++
.../src/main/resources/META-INF/LICENSE | 604 +++++++++++++
.../src/main/resources/META-INF/NOTICE | 92 ++
.../src/main/scripts/start_profiler_topology.sh | 22 +
.../zookeeper/event-time-test/profiler.json | 19 +
.../processing-time-test/profiler.json | 11 +
.../zookeeper/profile-with-stats/profiler.json | 12 +
.../storm/FixedFrequencyFlushSignalTest.java | 71 ++
.../metron/profiler/storm/HBaseEmitterTest.java | 118 +++
.../metron/profiler/storm/KafkaEmitterTest.java | 291 ++++++
.../profiler/storm/ProfileBuilderBoltTest.java | 378 ++++++++
.../profiler/storm/ProfileHBaseMapperTest.java | 93 ++
.../profiler/storm/ProfileSplitterBoltTest.java | 455 ++++++++++
.../integration/ConfigUploadComponent.java | 124 +++
.../storm/integration/MessageBuilder.java | 75 ++
.../integration/ProfilerIntegrationTest.java | 478 ++++++++++
.../src/test/resources/log4j.properties | 32 +
.../src/test/resources/telemetry.json | 100 +++
metron-analytics/metron-profiler/.gitignore | 1 -
metron-analytics/metron-profiler/README.md | 898 -------------------
metron-analytics/metron-profiler/pom.xml | 407 ---------
.../src/main/assembly/assembly.xml | 72 --
.../src/main/config/profiler.properties | 71 --
.../src/main/flux/profiler/remote.yaml | 218 -----
.../bolt/FixedFrequencyFlushSignal.java | 135 ---
.../metron/profiler/bolt/FlushSignal.java | 51 --
.../metron/profiler/bolt/HBaseEmitter.java | 73 --
.../metron/profiler/bolt/KafkaEmitter.java | 164 ----
.../metron/profiler/bolt/ManualFlushSignal.java | 54 --
.../profiler/bolt/ProfileBuilderBolt.java | 552 ------------
.../profiler/bolt/ProfileHBaseMapper.java | 117 ---
.../bolt/ProfileMeasurementEmitter.java | 59 --
.../profiler/bolt/ProfileSplitterBolt.java | 246 -----
.../src/main/resources/META-INF/LICENSE | 604 -------------
.../src/main/resources/META-INF/NOTICE | 92 --
.../src/main/scripts/start_profiler_topology.sh | 22 -
.../zookeeper/event-time-test/profiler.json | 19 -
.../processing-time-test/profiler.json | 11 -
.../zookeeper/profile-with-stats/profiler.json | 12 -
.../bolt/FixedFrequencyFlushSignalTest.java | 71 --
.../metron/profiler/bolt/HBaseEmitterTest.java | 120 ---
.../metron/profiler/bolt/KafkaEmitterTest.java | 291 ------
.../profiler/bolt/ProfileBuilderBoltTest.java | 378 --------
.../profiler/bolt/ProfileHBaseMapperTest.java | 93 --
.../profiler/bolt/ProfileSplitterBoltTest.java | 452 ----------
.../integration/ConfigUploadComponent.java | 124 ---
.../profiler/integration/MessageBuilder.java | 75 --
.../integration/ProfilerIntegrationTest.java | 478 ----------
.../src/test/resources/log4j.properties | 32 -
.../src/test/resources/telemetry.json | 100 ---
metron-analytics/pom.xml | 4 +-
.../common-services/METRON/CURRENT/metainfo.xml | 8 +-
.../packaging/docker/deb-docker/pom.xml | 14 +-
.../docker/rpm-docker/SPECS/metron.spec | 64 +-
.../packaging/docker/rpm-docker/pom.xml | 14 +-
.../metron-common/src/main/scripts/stellar | 3 +-
.../ZKConfigurationsCacheIntegrationTest.java | 2 +-
.../apache/metron/hbase/client/HBaseClient.java | 8 +-
.../metron/hbase/client/HBaseClientTest.java | 30 +-
pom.xml | 1 +
131 files changed, 11195 insertions(+), 7245 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/fbabfcad/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
[29/51] [abbrv] metron git commit: METRON-1807 Auto populate the
recommended values to some of the metron config parameters (MohanDV via
merrimanr) closes apache/metron#1227
Posted by ot...@apache.org.
METRON-1807 Auto populate the recommended values to some of the metron config parameters (MohanDV via merrimanr) closes apache/metron#1227
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/269b91d0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/269b91d0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/269b91d0
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 269b91d015c67b0ae49a78e5c457408e1b431c25
Parents: 891bf49
Author: MohanDV <mo...@gmail.com>
Authored: Mon Oct 8 11:20:53 2018 -0500
Committer: rmerriman <me...@gmail.com>
Committed: Mon Oct 8 11:20:53 2018 -0500
----------------------------------------------------------------------
.../METRON/CURRENT/configuration/metron-env.xml | 12 ++++++++----
.../common-services/METRON/CURRENT/service_advisor.py | 10 ++++++++++
2 files changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/269b91d0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index cdef7cf..81dda6c 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -130,17 +130,21 @@
<empty-value-valid>true</empty-value-valid>
</value-attributes>
</property>
- <property require-input = "true">
+ <property>
<name>storm_rest_addr</name>
<display-name>Storm Rest Server Address</display-name>
<description>URL of Storm UI (http://storm.ui.hostname:8744). If no protocol is provided, http is assumed.</description>
- <value></value>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
</property>
- <property require-input = "true">
+ <property >
<name>zeppelin_server_url</name>
<display-name>Zeppelin Server Address</display-name>
<description>URL of Zeppelin (zeppelin.server.addr:zeppelin.server.port)</description>
- <value></value>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
</property>
<property>
<name>metron_principal_name</name>
http://git-wip-us.apache.org/repos/asf/metron/blob/269b91d0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
index b008fc0..36cec7c 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
@@ -158,6 +158,16 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
putMetronEnvProperty = self.putProperty(configurations, "metron-env", services)
putMetronEnvProperty("zeppelin_server_url", zeppelinServerUrl)
+ #Suggest Zookeeper quorum
+ if "solr-cloud" in services["configurations"]:
+ zookeeperHost = self.getComponentHostNames(services, "ZOOKEEPER", "ZOOKEEPER_SERVER")[0]
+ zookeeperClientPort = services["configurations"]["zoo.cfg"]["properties"]["clientPort"]
+ solrZkDir = services["configurations"]["solr-cloud"]["properties"]["solr_cloud_zk_directory"]
+ solrZookeeperUrl = zookeeperHost + ":" + zookeeperClientPort + solrZkDir
+ putMetronEnvProperty = self.putProperty(configurations, "metron-env", services)
+ putMetronEnvProperty("solr_zookeeper_url", solrZookeeperUrl)
+
+
def validateSTORMSiteConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
# Determine if the cluster is secured
is_secured = self.isSecurityEnabled(services)
[39/51] [abbrv] metron git commit: METRON-1801 Allow Customization of
Elasticsearch Document ID (nickwallen) closes apache/metron#1218
Posted by ot...@apache.org.
METRON-1801 Allow Customization of Elasticsearch Document ID (nickwallen) closes apache/metron#1218
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/90c5e1d2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/90c5e1d2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/90c5e1d2
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 90c5e1d2896b8a95bd6c928e3dab96f1fd85167c
Parents: 9b67100
Author: nickwallen <ni...@nickallen.org>
Authored: Thu Oct 11 16:29:40 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Thu Oct 11 16:29:40 2018 -0400
----------------------------------------------------------------------
Upgrading.md | 23 ++-
.../METRON/CURRENT/configuration/metron-env.xml | 9 ++
.../CURRENT/package/scripts/metron_service.py | 5 +
.../package/scripts/params/params_linux.py | 1 +
.../METRON/CURRENT/themes/metron_theme.json | 10 ++
metron-platform/metron-elasticsearch/README.md | 37 ++++-
metron-platform/metron-elasticsearch/pom.xml | 17 ---
.../dao/ElasticsearchRetrieveLatestDao.java | 147 ++++++++++++-------
.../dao/ElasticsearchSearchDao.java | 28 +++-
.../elasticsearch/utils/ElasticsearchUtils.java | 100 +++++++++----
.../writer/ElasticsearchWriter.java | 39 +++--
.../elasticsearch/dao/ElasticsearchDaoTest.java | 10 +-
.../ElasticsearchSearchIntegrationTest.java | 20 ++-
.../indexing/dao/SearchIntegrationTest.java | 21 +--
.../integration/SolrSearchIntegrationTest.java | 19 +++
15 files changed, 341 insertions(+), 145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/Upgrading.md
----------------------------------------------------------------------
diff --git a/Upgrading.md b/Upgrading.md
index 2124ac5..c3b2a0f 100644
--- a/Upgrading.md
+++ b/Upgrading.md
@@ -16,9 +16,22 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
# Upgrading
+
This document constitutes a per-version listing of changes of
configuration which are non-backwards compatible.
+## 0.6.0 to 0.6.1
+
+### [METRON-1801 Allow Customization of Elasticsearch Document ID](https://issues.apache.org/jira/browse/METRON-1801)
+
+A global property named `es.document.id` was added to define the field from which the document ID is set when a message is indexed by Elasticsearch. To allow Elasticsearch to define its own document id, this property should be set to a blank or empty string. The client will not set the document ID and Elasticsearch will define its own. In most cases allowing Elasticsearch to define the document ID is the most performant option. This is now the default behavior.
+
+Metron versions 0.6.0 and earlier defined the document ID using the Metron GUID, which is a randomized UUID using Java's `UUID.randomUUID()`. Using a randomized UUID can negatively impact Elasticsearch indexing performance. To maintain backwards compatibility with legacy versions of Metron use the following global property setting.
+
+ ```
+ es.document.id = guid
+ ```
+
## 0.4.2 to 0.5.0
### [METRON-941: native PaloAlto parser corrupts message when having a comma in the payload](https://issues.apache.org/jira/browse/METRON-941)
@@ -89,7 +102,7 @@ For a more detailed description, please see metron-platform/metron-elasticsearch
### Description
-In the 0.4.2 release,
+In the 0.4.2 release,
## 0.3.1 to 0.4.0
@@ -107,7 +120,7 @@ This effectively limits the build environment to Docker supported [platforms](ht
#### Description
As of 0.3.0 the indexing configuration
-* Is held in the enrichment configuration for a sensor
+* Is held in the enrichment configuration for a sensor
* Has properties which control every writers (i.e. HDFS, solr or elasticsearch).
In the 0.3.1 release, this configuration has been broken out
@@ -136,7 +149,7 @@ You would create a file to configure each writer for sensor `foo` called `$METRO
"batchSize" : 100,
"enabled" : true
},
- "hdfs" : {
+ "hdfs" : {
"index" : "foo",
"batchSize" : 100,
"enabled" : true
@@ -151,7 +164,7 @@ You would create a file to configure each writer for sensor `foo` called `$METRO
As of 0.3.0, threat triage rules were defined as a simple Map associating a Stellar expression with a score.
As of 0.3.1, due to the fact that there may be many threat triage rules, we have made the rules more complex.
To help organize these, we have made the threat triage objects in their own right that contain optional name and optional comment fields.
-
+
This essentially makes the risk level rules slightly more complex. The format goes from:
```
"riskLevelRules" : {
@@ -169,7 +182,7 @@ to:
}
]
```
-
+
#### Migration
For every sensor enrichment configuration, you will need to migrate the `riskLevelRules` section
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index 81dda6c..0c7d295 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -122,6 +122,15 @@
</value-attributes>
</property>
<property>
+ <name>es_document_id</name>
+ <description>Message field containing the Elasticsearch document ID</description>
+ <value></value>
+ <display-name>Elasticsearch Document ID Source Field</display-name>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
<name>solr_zookeeper_url</name>
<value>{{zookeeper_quorum}}</value>
<description>Comma delimited list of Zookeeper Urls: zkhost1:2181,zkhost1:2181</description>
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
index 9d15e93..9b9d99b 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
@@ -83,6 +83,11 @@ def elasticsearch_global_config_patches():
"op": "add",
"path": "/es.date.format",
"value": "{{es_date_format}}"
+ },
+ {
+ "op": "add",
+ "path": "/es.document.id",
+ "value": "{{es_document_id}}"
}
"""
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 458a7be..316964a 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -101,6 +101,7 @@ es_url = ",".join([host + ":" + es_binary_port for host in es_host_list])
es_http_port = config['configurations']['metron-env']['es_http_port']
es_http_url = es_host_list[0] + ":" + es_http_port
es_date_format = config['configurations']['metron-env']['es_date_format']
+es_document_id = config['configurations']['metron-env']['es_document_id']
# hadoop params
stack_root = Script.get_stack_root()
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index 7e6c83a..34eff50 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -476,6 +476,10 @@
"subsection-name": "subsection-index-settings"
},
{
+ "config": "metron-env/es_document_id",
+ "subsection-name": "subsection-index-settings"
+ },
+ {
"config": "metron-env/solr_zookeeper_url",
"subsection-name": "subsection-index-settings"
},
@@ -949,6 +953,12 @@
}
},
{
+ "config": "metron-env/es_document_id",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
"config": "metron-env/solr_zookeeper_url",
"widget": {
"type": "text-field"
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/metron-platform/metron-elasticsearch/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/README.md b/metron-platform/metron-elasticsearch/README.md
index 177412e..25e15b4 100644
--- a/metron-platform/metron-elasticsearch/README.md
+++ b/metron-platform/metron-elasticsearch/README.md
@@ -59,6 +59,39 @@ For instance, an `es.date.format` of `yyyy.MM.dd.HH` would have the consequence
roll hourly, whereas an `es.date.format` of `yyyy.MM.dd` would have the consequence that the indices would
roll daily.
+### `es.document.id`
+
+This property sets the message field that is used to define the document ID when a message is indexed by Elasticsearch. By default, the client does not set the document ID and document ID generation is deferred to Elasticsearch.
+
+#### Option 1: Defer to Elasticsearch
+
+Value: (Undefined, blank or empty string)
+
+* This option allows Elasticsearch to generate the document ID.
+* The client will not set a document ID.
+* In most cases this is the most performant option.
+* This is the default behavior.
+
+#### Option 2: Legacy Compatibility
+
+Value: guid
+
+* Metron versions 0.6.0 and earlier defined the document ID using the Metron GUID, which is a randomized UUID using Java's `UUID.randomUUID()`.
+* Using a randomized UUID can negatively impact Elasticsearch indexing performance.
+* To maintain backwards compatibility with legacy versions of Metron, set the value to `guid`.
+ ```
+ es.document.id = guid
+ ```
+
+#### Option 3: Custom Document ID
+
+* Advanced users can define a custom document ID.
+* Create an enrichment that defines a new message field; for example one called `my_document_id`. Use this field to set the document ID. This will set the document ID to the value of the message field `my_document_id`.
+ ```
+ es.document.id = my_document_id
+ ```
+* If a message does not contain the `es.document.id` field, a warning is issued and no document ID is set by the client.
+
## Upgrading to 5.6.2
Users should be prepared to re-index when migrating from Elasticsearch 2.3.3 to 5.6.2. There are a number of template changes, most notably around
@@ -321,7 +354,7 @@ We'll want to put the template back into Elasticsearch:
curl -XPUT "http://${ELASTICSEARCH}:9200/_template/${SENSOR}_index" -d @${SENSOR}.template
```
-To update existing indexes, update Elasticsearch mappings with the new field for each sensor.
+To update existing indexes, update Elasticsearch mappings with the new field for each sensor.
```
curl -XPUT "http://${ELASTICSEARCH}:9200/${SENSOR}_index*/_mapping/${SENSOR}_doc" -d '
@@ -340,7 +373,7 @@ rm ${SENSOR}.template
The stock set of Elasticsearch templates for bro, snort, yaf, error index and meta index are installed automatically during the first time install and startup of Metron Indexing service.
-It is possible that Elasticsearch service is not available when the Metron Indexing Service startup, in that case the Elasticsearch template will not be installed.
+It is possible that Elasticsearch service is not available when the Metron Indexing Service startup, in that case the Elasticsearch template will not be installed.
For such a scenario, an Admin can have the template installed in two ways:
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/metron-platform/metron-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index adc601a..edec1bb 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -206,24 +206,7 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- <version>${global_log4j_core_version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>${global_log4j_core_version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava-testlib</artifactId>
- <version>${global_guava_version}</version>
- <scope>test</scope>
- </dependency>
</dependencies>
-
<build>
<plugins>
<plugin>
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRetrieveLatestDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRetrieveLatestDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRetrieveLatestDao.java
index f6bfeda..2f00b68 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRetrieveLatestDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchRetrieveLatestDao.java
@@ -20,27 +20,36 @@ package org.apache.metron.elasticsearch.dao;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Optional;
-import java.util.function.Function;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.common.Constants;
+import org.apache.metron.elasticsearch.utils.ElasticsearchUtils;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.index.query.IdsQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
public class ElasticsearchRetrieveLatestDao implements RetrieveLatestDao {
+ protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private TransportClient transportClient;
public ElasticsearchRetrieveLatestDao(TransportClient transportClient) {
@@ -49,7 +58,7 @@ public class ElasticsearchRetrieveLatestDao implements RetrieveLatestDao {
@Override
public Document getLatest(String guid, String sensorType) {
- Optional<Document> doc = searchByGuid(guid, sensorType, hit -> toDocument(guid, hit));
+ Optional<Document> doc = searchByGuid(guid, sensorType, hit -> toDocument(hit));
return doc.orElse(null);
}
@@ -61,26 +70,19 @@ public class ElasticsearchRetrieveLatestDao implements RetrieveLatestDao {
guids.add(getRequest.getGuid());
sensorTypes.add(getRequest.getSensorType());
}
- List<Document> documents = searchByGuids(
- guids,
- sensorTypes,
- hit -> {
- Long ts = 0L;
- String doc = hit.getSourceAsString();
- String sourceType = Iterables.getFirst(Splitter.on("_doc").split(hit.getType()), null);
- try {
- return Optional.of(new Document(doc, hit.getId(), sourceType, ts));
- } catch (IOException e) {
- throw new IllegalStateException("Unable to retrieve latest: " + e.getMessage(), e);
- }
- }
-
- );
- return documents;
+ return searchByGuids(guids, sensorTypes, hit -> toDocument(hit));
}
- <T> Optional<T> searchByGuid(String guid, String sensorType,
- Function<SearchHit, Optional<T>> callback) {
+ /**
+ * Search for a Metron GUID.
+ *
+ * @param guid The Metron GUID to search for.
+ * @param sensorType The sensor type to include in the search.
+ * @param callback A callback that transforms a search hit to a search result.
+ * @param <T> The type of search results expected.
+ * @return All documents with the given Metron GUID and sensor type.
+ */
+ <T> Optional<T> searchByGuid(String guid, String sensorType, Function<SearchHit, Optional<T>> callback) {
Collection<String> sensorTypes = sensorType != null ? Collections.singleton(sensorType) : null;
List<T> results = searchByGuids(Collections.singleton(guid), sensorTypes, callback);
if (results.size() > 0) {
@@ -91,48 +93,69 @@ public class ElasticsearchRetrieveLatestDao implements RetrieveLatestDao {
}
/**
- * Return the search hit based on the UUID and sensor type.
- * A callback can be specified to transform the hit into a type T.
- * If more than one hit happens, the first one will be returned.
+ * Search for one or more Metron GUIDs.
+ *
+ * @param guids The Metron GUIDs to search by.
+ * @param sensorTypes The sensor types to include in the search.
+ * @param callback A callback that transforms a search hit to a search result.
+ * @param <T> The type of expected results.
+ * @return All documents with the given Metron GUID and sensor type.
*/
- <T> List<T> searchByGuids(Collection<String> guids, Collection<String> sensorTypes,
- Function<SearchHit, Optional<T>> callback) {
+ <T> List<T> searchByGuids(Collection<String> guids,
+ Collection<String> sensorTypes,
+ Function<SearchHit, Optional<T>> callback) {
if (guids == null || guids.isEmpty()) {
return Collections.emptyList();
}
- QueryBuilder query = null;
- IdsQueryBuilder idsQuery;
- if (sensorTypes != null) {
- String[] types = sensorTypes.stream().map(sensorType -> sensorType + "_doc")
- .toArray(String[]::new);
- idsQuery = QueryBuilders.idsQuery(types);
- } else {
- idsQuery = QueryBuilders.idsQuery();
- }
- for (String guid : guids) {
- query = idsQuery.addIds(guid);
+ // search by metron's GUID field
+ QueryBuilder query = boolQuery().must(termsQuery(Constants.GUID, guids));
+ SearchRequestBuilder request = transportClient
+ .prepareSearch()
+ .setQuery(query)
+ .setSize(guids.size());
+
+ if(sensorTypes != null) {
+ request.setTypes(toDocTypes(sensorTypes));
}
- SearchRequestBuilder request = transportClient.prepareSearch()
- .setQuery(query)
- .setSize(guids.size());
- org.elasticsearch.action.search.SearchResponse response = request.get();
- SearchHits hits = response.getHits();
+ // transform the search hits to results
+ SearchHits hits = request.get().getHits();
+ return getResults(hits, callback);
+ }
+
+ /**
+ * Retrieve search results.
+ *
+ * @param hits The search hit.
+ * @param callback A callback function that transforms search hits to search results.
+ * @param <T> The execpted type of search result.
+ * @return The search results.
+ */
+ private <T> List<T> getResults(SearchHits hits, Function<SearchHit, Optional<T>> callback) {
List<T> results = new ArrayList<>();
+
for (SearchHit hit : hits) {
Optional<T> result = callback.apply(hit);
if (result.isPresent()) {
results.add(result.get());
}
}
+
return results;
}
- private Optional<Document> toDocument(final String guid, SearchHit hit) {
+ /**
+ * Transforms a {@link SearchHit} to a {@link Document}.
+ *
+ * @param hit The result of a search.
+ * @return An optional {@link Document}.
+ */
+ private Optional<Document> toDocument(SearchHit hit) {
Long ts = 0L;
String doc = hit.getSourceAsString();
String sourceType = toSourceType(hit.getType());
+ String guid = ElasticsearchUtils.getGUID(hit);
try {
return Optional.of(new Document(doc, guid, sourceType, ts));
} catch (IOException e) {
@@ -148,4 +171,26 @@ public class ElasticsearchRetrieveLatestDao implements RetrieveLatestDao {
private String toSourceType(String docType) {
return Iterables.getFirst(Splitter.on("_doc").split(docType), null);
}
+
+ /**
+ * Returns the doc types for a given collection of sensor types.
+ *
+ * @param sensorTypes The sensor types.
+ * @return The doc types associated with each sensor.
+ */
+ private String[] toDocTypes(Collection<String> sensorTypes) {
+ String[] result;
+
+ if(sensorTypes != null && sensorTypes.size() > 0) {
+ result = sensorTypes
+ .stream()
+ .map(sensorType -> sensorType + "_doc")
+ .toArray(String[]::new);
+
+ } else {
+ result = new String[0];
+ }
+
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
index 5cd0a4d..c944578 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
@@ -33,6 +33,8 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
+
+import org.apache.metron.common.Constants;
import org.apache.metron.elasticsearch.utils.ElasticsearchUtils;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.search.FieldType;
@@ -74,6 +76,8 @@ import org.elasticsearch.search.sort.FieldSortBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.metron.common.Constants.GUID;
+
public class ElasticsearchSearchDao implements SearchDao {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -284,19 +288,29 @@ public class ElasticsearchSearchDao implements SearchDao {
.toArray(value -> new String[indices.size()]);
}
- private SearchResult getSearchResult(SearchHit searchHit, List<String> fields) {
- SearchResult searchResult = new SearchResult();
- searchResult.setId(searchHit.getId());
+ /**
+ * Transforms a {@link SearchHit} to a {@link SearchResult}.
+ *
+ * @param searchHit The result of a search request.
+ * @param fieldsToKeep The source fields to keep. All others are excluded. If null, all source fields are kept.
+ * @return A search result.
+ */
+ private SearchResult getSearchResult(SearchHit searchHit, List<String> fieldsToKeep) {
+
Map<String, Object> source;
- if (fields != null) {
- Map<String, Object> resultSourceAsMap = searchHit.getSourceAsMap();
+ if (fieldsToKeep != null) {
source = new HashMap<>();
- fields.forEach(field -> {
- source.put(field, resultSourceAsMap.get(field));
+ // must always keep the GUID
+ fieldsToKeep.add(Constants.GUID);
+ fieldsToKeep.forEach(field -> {
+ source.put(field, searchHit.getSourceAsMap().get(field));
});
} else {
source = searchHit.getSource();
}
+
+ SearchResult searchResult = new SearchResult();
+ searchResult.setId(ElasticsearchUtils.getGUID(searchHit));
searchResult.setSource(source);
searchResult.setScore(searchHit.getScore());
searchResult.setIndex(searchHit.getIndex());
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
index 98dc66d..31503fa 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
@@ -17,26 +17,9 @@
*/
package org.apache.metron.elasticsearch.utils;
-import static java.lang.String.format;
-
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.utils.HDFSUtils;
@@ -53,10 +36,29 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.transport.client.PreBuiltTransportClient;
+import org.elasticsearch.search.SearchHit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.lang.String.format;
+import static org.apache.metron.common.Constants.GUID;
+
public class ElasticsearchUtils {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -65,6 +67,14 @@ public class ElasticsearchUtils {
private static final String USERNAME_CONFIG_KEY = "es.xpack.username";
private static final String TRANSPORT_CLIENT_USER_KEY = "xpack.security.user";
+ /**
+ * Defines which message field, the document identifier is set to.
+ *
+ * <p>If defined, the value of the specified message field is set as the Elasticsearch doc ID. If
+ * this field is undefined or blank, then the document identifier is not set.
+ */
+ public static final String DOC_ID_SOURCE_FIELD = "es.document.id";
+ public static final String DOC_ID_SOURCE_FIELD_DEFAULT = "";
private static ThreadLocal<Map<String, SimpleDateFormat>> DATE_FORMAT_CACHE
= ThreadLocal.withInitial(() -> new HashMap<>());
@@ -373,16 +383,48 @@ public class ElasticsearchUtils {
* @param searchResponse An Elasticsearch SearchHit to be converted.
* @return The list of SearchResults for the SearchHit
*/
- protected static List<SearchResult> getSearchResults(
- org.elasticsearch.action.search.SearchResponse searchResponse) {
- return Arrays.stream(searchResponse.getHits().getHits()).map(searchHit -> {
- SearchResult searchResult = new SearchResult();
- searchResult.setId(searchHit.getId());
- searchResult.setSource(searchHit.getSource());
- searchResult.setScore(searchHit.getScore());
- searchResult.setIndex(searchHit.getIndex());
- return searchResult;
- }
- ).collect(Collectors.toList());
+ protected static List<SearchResult> getSearchResults(org.elasticsearch.action.search.SearchResponse searchResponse) {
+ SearchHit[] searchHits = searchResponse.getHits().getHits();
+ return Arrays.stream(searchHits)
+ .map(hit -> toSearchResult(hit))
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Transforms a {@link SearchHit} to a {@link SearchResult}.
+ *
+ * @param searchHit The search hit to transform.
+ * @return A {@link SearchResult} representing the {@link SearchHit}.
+ */
+ protected static SearchResult toSearchResult(SearchHit searchHit) {
+ SearchResult searchResult = new SearchResult();
+ searchResult.setId(getGUID(searchHit));
+ searchResult.setSource(searchHit.getSource());
+ searchResult.setScore(searchHit.getScore());
+ searchResult.setIndex(searchHit.getIndex());
+ return searchResult;
+ }
+
+ /**
+ * Retrieves the Metron GUID from a {@link SearchHit}.
+ *
+ * @param searchHit The search hit containing a Metron GUID.
+ * @return The Metron GUID.
+ */
+ public static String getGUID(SearchHit searchHit) {
+ String guid;
+ if(searchHit.hasSource() && searchHit.getSource().containsKey(GUID)) {
+ guid = (String) searchHit.getSource().get(GUID);
+
+ } else if(!searchHit.hasSource()) {
+ String template = "No source found, has it been disabled in the mapping? index=%s, docId=%s";
+ throw new IllegalStateException(String.format(template, searchHit.getIndex(), searchHit.getId()));
+
+ } else {
+ String template = "Missing expected field; field=%s, index=%s, docId=%s";
+ throw new IllegalStateException(String.format(template, GUID, searchHit.getIndex(), searchHit.getId()));
+ }
+
+ return guid;
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
index 4b8dd08..9a18e8c 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
@@ -17,6 +17,8 @@
*/
package org.apache.metron.elasticsearch.writer;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.field.FieldNameConverter;
@@ -43,6 +45,9 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import static org.apache.metron.elasticsearch.utils.ElasticsearchUtils.DOC_ID_SOURCE_FIELD;
+import static org.apache.metron.elasticsearch.utils.ElasticsearchUtils.DOC_ID_SOURCE_FIELD_DEFAULT;
+
/**
* A {@link BulkMessageWriter} that writes messages to Elasticsearch.
*/
@@ -71,36 +76,48 @@ public class ElasticsearchWriter implements BulkMessageWriter<JSONObject>, Seria
@Override
public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception {
-
- // fetch the field name converter for this sensor type
+ // writer settings
FieldNameConverter fieldNameConverter = FieldNameConverters.create(sensorType, configurations);
-
final String indexPostfix = dateFormat.format(new Date());
+ final String indexName = ElasticsearchUtils.getIndexName(sensorType, indexPostfix, configurations);
+ final String docType = sensorType + "_doc";
+ final String docIdSourceField = (String) configurations.getGlobalConfig().getOrDefault(DOC_ID_SOURCE_FIELD, DOC_ID_SOURCE_FIELD_DEFAULT);
+
BulkRequestBuilder bulkRequest = client.prepareBulk();
for(JSONObject message: messages) {
+ // clone the message to use as the document that will be indexed
JSONObject esDoc = new JSONObject();
for(Object k : message.keySet()){
copyField(k.toString(), message, esDoc, fieldNameConverter);
}
- String indexName = ElasticsearchUtils.getIndexName(sensorType, indexPostfix, configurations);
- IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName, sensorType + "_doc");
- indexRequestBuilder = indexRequestBuilder.setSource(esDoc.toJSONString());
- String guid = (String)esDoc.get(Constants.GUID);
- if(guid != null) {
- indexRequestBuilder.setId(guid);
+ IndexRequestBuilder indexRequestBuilder = client
+ .prepareIndex(indexName, docType)
+ .setSource(esDoc.toJSONString());
+
+ // set the document identifier
+ if(StringUtils.isNotBlank(docIdSourceField)) {
+ String docId = (String) esDoc.get(docIdSourceField);
+ if(docId != null) {
+ indexRequestBuilder.setId(docId);
+ } else {
+ LOG.warn("Message is missing document ID source field; document ID not set; sourceField={}", docIdSourceField);
+ }
}
- Object ts = esDoc.get("timestamp");
+ // set the document timestamp, if one exists
+ Object ts = esDoc.get(Constants.Fields.TIMESTAMP.getName());
if(ts != null) {
- indexRequestBuilder = indexRequestBuilder.setTimestamp(ts.toString());
+ indexRequestBuilder.setTimestamp(ts.toString());
}
bulkRequest.add(indexRequestBuilder);
}
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
+ LOG.info("Wrote {} message(s) to Elasticsearch; sensorType={}, index={}, docType={}, took={}",
+ ArrayUtils.getLength(bulkResponse.getItems()), sensorType, indexName, docType, bulkResponse.getTook().format());
return buildWriteReponse(tuples, bulkResponse);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
index 6c3c327..a2c8de5 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
@@ -29,6 +29,8 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+
+import org.apache.metron.common.Constants;
import org.apache.metron.elasticsearch.utils.ElasticsearchUtils;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.search.FieldType;
@@ -57,16 +59,20 @@ public class ElasticsearchDaoTest {
// setup the mock search hits
SearchHit hit1 = mock(SearchHit.class);
- when(hit1.getId()).thenReturn("id1");
+ when(hit1.getId()).thenReturn("docId1");
+ when(hit1.hasSource()).thenReturn(true);
when(hit1.getSource()).thenReturn(new HashMap<String, Object>() {{
put("field", "value1");
+ put(Constants.GUID, "id1");
}});
when(hit1.getScore()).thenReturn(0.1f);
SearchHit hit2 = mock(SearchHit.class);
- when(hit2.getId()).thenReturn("id2");
+ when(hit2.getId()).thenReturn("docId2");
+ when(hit2.hasSource()).thenReturn(true);
when(hit2.getSource()).thenReturn(new HashMap<String, Object>() {{
put("field", "value2");
+ put(Constants.GUID, "id2");
}});
when(hit2.getScore()).thenReturn(0.2f);
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 1d2d48e..df41cba 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -168,8 +168,6 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
dao.search(request);
}
-
-
@Override
public void returns_column_metadata_for_specified_indices() throws Exception {
// getColumnMetadata with only bro
@@ -261,6 +259,24 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
Assert.assertEquals("data 1", results.get(0).getSource().get("ttl"));
}
+ @Test
+ public void queries_fields() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class);
+ SearchResponse response = getIndexDao().search(request);
+ Assert.assertEquals(10, response.getTotal());
+
+ List<SearchResult> results = response.getResults();
+ Assert.assertEquals(10, response.getResults().size());
+
+ // validate the source fields contained in the search response
+ for (int i = 0; i < 10; ++i) {
+ Map<String, Object> source = results.get(i).getSource();
+ Assert.assertNotNull(source);
+ Assert.assertNotNull(source.get(Constants.Fields.SRC_ADDR.getName()));
+ Assert.assertNotNull(source.get(Constants.GUID));
+ }
+ }
+
@Override
protected String getSourceTypeField() {
return Constants.SENSOR_TYPE.replace('.', ':');
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index cfe5752..43e8c85 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
@@ -897,24 +898,6 @@ public abstract class SearchIntegrationTest {
}
@Test
- public void queries_fields() throws Exception {
- SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class);
- SearchResponse response = getIndexDao().search(request);
- Assert.assertEquals(10, response.getTotal());
- List<SearchResult> results = response.getResults();
- for (int i = 0; i < 5; ++i) {
- Map<String, Object> source = results.get(i).getSource();
- Assert.assertEquals(1, source.size());
- Assert.assertNotNull(source.get("ip_src_addr"));
- }
- for (int i = 5; i < 10; ++i) {
- Map<String, Object> source = results.get(i).getSource();
- Assert.assertEquals(1, source.size());
- Assert.assertNotNull(source.get("ip_src_addr"));
- }
- }
-
- @Test
public void sort_by_guid() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(sortByGuidQuery, SearchRequest.class);
SearchResponse response = getIndexDao().search(request);
@@ -923,7 +906,7 @@ public abstract class SearchIntegrationTest {
for (int i = 0; i < 5; ++i) {
Map<String, Object> source = results.get(i).getSource();
Assert.assertEquals(1, source.size());
- Assert.assertEquals(source.get("guid"), "bro_" + (i + 1));
+ Assert.assertEquals(source.get(Constants.GUID), "bro_" + (i + 1));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/90c5e1d2/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
index 84f2222..58e9db4 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
@@ -33,6 +34,7 @@ import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.integration.InMemoryComponent;
import org.apache.metron.solr.dao.SolrDao;
import org.apache.metron.solr.integration.components.SolrComponent;
@@ -224,6 +226,23 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
}
@Test
+ public void queries_fields() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class);
+ SearchResponse response = getIndexDao().search(request);
+ Assert.assertEquals(10, response.getTotal());
+
+ List<SearchResult> results = response.getResults();
+ Assert.assertEquals(10, response.getResults().size());
+
+ // validate the source fields contained in the search response
+ for (int i = 0; i < 10; ++i) {
+ Map<String, Object> source = results.get(i).getSource();
+ Assert.assertNotNull(source);
+ Assert.assertNotNull(source.get(Constants.Fields.SRC_ADDR.getName()));
+ }
+ }
+
+ @Test
public void different_type_filter_query() throws Exception {
thrown.expect(InvalidSearchException.class);
SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFilterQuery, SearchRequest.class);