You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2017/09/21 15:59:13 UTC
[3/3] metron git commit: METRON-1188: Ambari global configuration
management (mmiklavc) closes apache/metron#760
METRON-1188: Ambari global configuration management (mmiklavc) closes apache/metron#760
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c18faaa9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c18faaa9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c18faaa9
Branch: refs/heads/master
Commit: c18faaa94cd9b99bd88612899b5baed897a7681c
Parents: 87ff7b7
Author: mmiklavc <mi...@gmail.com>
Authored: Thu Sep 21 09:58:38 2017 -0600
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Thu Sep 21 09:58:38 2017 -0600
----------------------------------------------------------------------
dependencies_with_url.csv | 2 +-
metron-deployment/packaging/ambari/README.md | 75 +++-
.../configuration/metron-enrichment-env.xml | 24 ++
.../configuration/metron-indexing-env.xml | 8 +-
.../package/scripts/enrichment_commands.py | 53 +--
.../package/scripts/enrichment_master.py | 8 +-
.../package/scripts/indexing_commands.py | 44 +--
.../CURRENT/package/scripts/indexing_master.py | 7 +-
.../CURRENT/package/scripts/metron_service.py | 127 +++++--
.../package/scripts/params/params_linux.py | 15 +-
.../package/scripts/params/status_params.py | 20 +-
.../CURRENT/package/scripts/parser_commands.py | 11 +-
.../CURRENT/package/scripts/parser_master.py | 6 +-
.../package/scripts/profiler_commands.py | 31 +-
.../CURRENT/package/scripts/profiler_master.py | 12 +-
.../CURRENT/package/scripts/rest_commands.py | 12 +-
.../CURRENT/package/templates/global.json.j2 | 8 -
.../METRON/CURRENT/themes/metron_theme.json | 8 +-
.../docker/rpm-docker/SPECS/metron.spec | 3 +
.../impl/GlobalConfigServiceImplTest.java | 31 +-
metron-platform/metron-common/README.md | 158 +++++++-
metron-platform/metron-common/pom.xml | 19 +
.../src/main/assembly/assembly.xml | 12 +
.../src/main/config/zookeeper/global.json | 8 +
.../metron/common/cli/ConfigurationManager.java | 274 +++++++++++---
.../common/configuration/ConfigurationType.java | 13 +-
.../common/configuration/Configurations.java | 4 +-
.../configuration/ConfigurationsUtils.java | 305 +++++++++++++---
.../configuration/EnrichmentConfigurations.java | 2 +-
.../configuration/IndexingConfigurations.java | 2 +-
.../configuration/ParserConfigurations.java | 2 +-
.../profiler/ProfilerConfigurations.java | 2 +-
.../apache/metron/common/utils/JSONUtils.java | 71 +++-
.../apache/metron/common/utils/StringUtils.java | 14 +-
.../bolt/ConfiguredEnrichmentBoltTest.java | 8 +-
.../common/bolt/ConfiguredParserBoltTest.java | 8 +-
.../ConfigurationManagerIntegrationTest.java | 356 ++++++++++++++++++-
.../common/cli/ConfigurationsUtilsTest.java | 92 -----
.../configuration/ConfigurationsUtilsTest.java | 170 +++++++++
.../metron/common/utils/JSONUtilsTest.java | 101 +++++-
.../src/main/config/enrichment.properties.j2 | 12 +-
.../integration/EnrichmentIntegrationTest.java | 44 +--
metron-platform/metron-indexing/pom.xml | 19 -
.../apache/metron/indexing/dao/HBaseDao.java | 3 +-
.../apache/metron/indexing/dao/IndexDao.java | 17 +-
.../metron/integration/utils/TestUtils.java | 82 +++++
.../SolrIndexingIntegrationTest.java | 2 +-
47 files changed, 1799 insertions(+), 506 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 9ebb9e4..b95f25b 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -18,7 +18,7 @@ com.esotericsoftware.minlog:minlog:jar:1.2:compile,New BSD License,http://code.g
com.esotericsoftware.minlog:minlog:jar:1.3.0:compile,New BSD License,http://code.google.com/p/minlog/
com.esotericsoftware:minlog:jar:1.3.0:compile,New BSD License,http://code.google.com/p/minlog/
com.esotericsoftware:reflectasm:jar:1.10.1:compile,New BSD License,http://code.google.com/p/minlog/
-com.flipkart.zjsonpatch:zjsonpatch:jar:0.3.1:compile,Apache v2, https://github.com/flipkart-incubator/zjsonpatch
+com.flipkart.zjsonpatch:zjsonpatch:jar:0.3.4:compile,Apache v2, https://github.com/flipkart-incubator/zjsonpatch
com.google.protobuf:protobuf-java:jar:2.5.0:compile,New BSD license,http://code.google.com/p/protobuf
com.google.protobuf:protobuf-java:jar:2.6.1:compile,New BSD license,http://code.google.com/p/protobuf
com.jcraft:jsch:jar:0.1.42:compile,BSD,http://www.jcraft.com/jsch/
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/README.md b/metron-deployment/packaging/ambari/README.md
index e823e3c..af511b4 100644
--- a/metron-deployment/packaging/ambari/README.md
+++ b/metron-deployment/packaging/ambari/README.md
@@ -1,4 +1,17 @@
# Ambari Management Pack Development
+
+## Table of Contents
+
+* [Overview](#overview)
+* [Ambari Zookeeper Config Lifecycle](#ambari-zookeeper-config-lifecycle)
+* [Adding a new property](#adding-a-new-property)
+* [How to identify errors in MPack changes](#how-to-identify-errors-in-mpack-changes)
+* [Testing changes without cycling Vagrant build](#testing-changes-without-cycling-vagrant-build)
+* [Configuration involving dependency services](#configuration-involving-dependency-services)
+* [Kerberos](#kerberos)
+* [Best practices](#best-practices)
+
+## Overview
Typically, Ambari Management Pack development will be done in the Vagrant environments. These instructions are specific to Vagrant, but can be adapted for other environemnts (e.g. make sure to be on the correct nodes for server vs agent files)
There is an `mpack.json` file which describes what services the mpack will contains, versions, etc.
@@ -36,6 +49,38 @@ The layout of `/addon-services/METRON.CURRENT` is
* `role_command_order.json`
* Defines the order of service startup and other actions relative to each other.
+## Ambari Zookeeper Config Lifecycle
+The source of truth for Metron configuration resides in Zookeeper as a series of JSON documents. There are actually multiple locations that you can populate the Zookeeper configs from:
+- $METRON_HOME/config/zookeeper
+- Stellar REPL
+- Management UI
+- Ambari
+
+Any change that works with Metron configuration stored in Zookeeper should always pull the latest config from Zookeeper first before making any changes and publishing them back up to avoid
+potentially overwriting the latest configs with stale data. The lifecycle for configuration management with Ambari is as follows:
+- First start/install for parsers, enrichment, indexing, or the profiler
+ - Perform a PUSH from local disk to Zookeeper
+- All component start/restart operations for parsers, enrichment, indexing, or the profiler
+ - Perform a JSON patch (RFC 6902) on the configs in Zookeeper and push the patched, pretty formatted config up to Zookeeper. Patching is currently applicable to to global.json only.
+ - Pull all configs to the local file system.
+
+The patching mechanism we introduced to Ambari for managing the global config is by default additive in nature, though it will also overwrite any existing values with the latest values from Ambari.
+This means that you can add any properties to the global config outside of Ambari you like, provided you are not modifying values explicitly managed by Ambari. This is important because you will not
+get any errors when modifying the configuration outside of Ambari. Instead, if you modified es.ip and changed global.json outsided Ambari, you would not see this change in Ambari. Meanwhile, the
+indexing topology would be using the new value stored in Zookeeper. Remember, Zookeeper is the source of truth. If you were to restart a Metron topology component via Ambari, that es.ip property
+would now be set back to the value stored in Ambari. So in summary, if it's managed by Ambari, only change that value via Ambari. If it's not, then feel free to use whatever mechanism you like,
+just be sure to pull the latest config from Zookeeper before making any changes. We make no effort to handle race conditions. It's last commit wins. Below are the global config properties managed
+by Ambari:
+- es.clustername
+- es.ip
+- es.date.format
+- parser.error.topic
+- update.hbase.table
+- update.hbase.cf
+- profiler.client.period.duration
+- profiler.client.period.duration.units
+
+
## Adding a new property
1. Add the property to the appropriate `*-env.xml` file found in `METRON.CURRENT/configuration`.
```
@@ -61,7 +106,8 @@ Afterwards, in `params_linux.py`, reference the new property:
```
new_property = status_params.new_property
```
-This behavior is because Ambari doesn't send all parameters to the status, so it needs to be explicitly provided.
+This behavior is because Ambari doesn't send all parameters to the status, so it needs to be explicitly provided. Also note that status_params.py parameters are not automatically pulled into the params_linux.py namespace, so we explicitly choose the variables to include.
+ See https://docs.python.org/2/howto/doanddont.html#at-module-level for more info.
4. Ambari master services can then import the params:
@@ -118,6 +164,33 @@ hdfs_grok_patterns_dir = format("{metron_apps_hdfs_dir}/patterns")
The `format` function is a special Ambari function that will let you create a string with properties in curly braces replaced by their values.
In this case, `hdfs_grok_patterns_dir` will be `/apps/metron/patterns` and will be what we now follow in this example.
+Note: There are instances where we deviate a bit from this pattern and use format(format("{{some_prop}}")). This is not a mistake. Ambari does not natively support nested properties, and this was a workaround that accomplished being
+able to initialize values from env.xml files and have them also available to users as Ambari properties later. Here's an example from params_linux.py:
+```
+metron_apps_indexed_hdfs_dir = format(format(config['configurations']['metron-indexing-env']['metron_apps_indexed_hdfs_dir']))
+```
+and the relavant properties in metron-env.xml and metron-indexing-env.xml
+
+metron-env.xml
+```
+<property>
+ <name>metron_apps_hdfs_dir</name>
+ <value>/apps/metron</value>
+ <description>Metron apps HDFS dir</description>
+ <display-name>Metron apps HDFS dir</display-name>
+</property>
+```
+
+metron-indexing-env.xml
+```
+<property>
+ <name>metron_apps_indexed_hdfs_dir</name>
+ <value>{{metron_apps_hdfs_dir}}/indexing/indexed</value>
+ <description>Indexing bolts will write to this HDFS directory</description>
+ <display-name>Metron apps indexed HDFS dir</display-name>
+</property>
+```
+
#### Importing the parameters into scripts
`hdfs_grok_patterns_dir` is used in `METRON.CURRENT/package/scripts/parser_commands.py`, but before we can reference it, we'll need the params available to `parser_commands.py`
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
index feff54a..9737660 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
@@ -82,6 +82,30 @@
<display-name>Threat Intel Error Topic</display-name>
</property>
<property>
+ <name>enrichment_hbase_table</name>
+ <value>enrichment</value>
+ <description>The HBase table which will hold enrichment data</description>
+ <display-name>Enrichment HBase Table</display-name>
+ </property>
+ <property>
+ <name>enrichment_hbase_cf</name>
+ <value>t</value>
+ <description>The HBase column family which will hold enrichment data in HBase.</description>
+ <display-name>HBase Table Column Family</display-name>
+ </property>
+ <property>
+ <name>threatintel_hbase_table</name>
+ <value>threatintel</value>
+ <description>The HBase table which will hold threatintel data</description>
+ <display-name>Threatintel HBase Table</display-name>
+ </property>
+ <property>
+ <name>threatintel_hbase_cf</name>
+ <value>t</value>
+ <description>The HBase column family which will hold threatintel data in HBase.</description>
+ <display-name>HBase Table Column Family</display-name>
+ </property>
+ <property>
<name>enrichment_workers</name>
<description>Number of Workers for the Enrichment Topology</description>
<value>1</value>
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
index 6abbe77..d66ef20 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
@@ -61,15 +61,15 @@
<display-name>Indexing Writer Class Name</display-name>
</property>
<property>
- <name>update_table</name>
- <description>The HBase table which will hold edits to indexed data</description>
+ <name>update_hbase_table</name>
<value>metron_update</value>
+ <description>The HBase table which will hold edits to indexed data</description>
<display-name>Indexing Update Table</display-name>
</property>
<property>
- <name>update_cf</name>
- <description>The HBase column family which will hold edits to indexed data</description>
+ <name>update_hbase_cf</name>
<value>t</value>
+ <description>The HBase column family which will hold edits to indexed data</description>
<display-name>Indexing Update Column Family</display-name>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
index 1df448b..794b6a5 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
@@ -17,6 +17,7 @@ limitations under the License.
import os
import time
+from datetime import datetime
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute, File
@@ -52,49 +53,29 @@ class EnrichmentCommands:
def is_kafka_acl_configured(self):
return self.__kafka_acl_configured
- def set_kafka_configured(self):
- Logger.info("Setting Kafka Configured to True for enrichment")
- File(self.__params.enrichment_kafka_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
-
- def set_kafka_acl_configured(self):
- Logger.info("Setting Kafka ACL Configured to True for enrichment")
- File(self.__params.enrichment_kafka_acl_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
-
def is_hbase_configured(self):
return self.__hbase_configured
def is_hbase_acl_configured(self):
return self.__hbase_acl_configured
+ def is_geo_configured(self):
+ return self.__geo_configured
+
+ def set_kafka_configured(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.enrichment_kafka_configured_flag_file, "Setting Kafka configured to True for enrichment")
+
+ def set_kafka_acl_configured(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.enrichment_kafka_acl_configured_flag_file, "Setting Kafka ACL configured to True for enrichment")
+
def set_hbase_configured(self):
- Logger.info("Setting HBase Configured to True for enrichment")
- File(self.__params.enrichment_hbase_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
+ metron_service.set_configured(self.__params.metron_user, self.__params.enrichment_hbase_configured_flag_file, "Setting HBase configured to True for enrichment")
def set_hbase_acl_configured(self):
- Logger.info("Setting HBase ACL Configured to True for enrichment")
- File(self.__params.enrichment_hbase_acl_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
-
- def is_geo_configured(self):
- return self.__geo_configured
+ metron_service.set_configured(self.__params.metron_user, self.__params.enrichment_hbase_acl_configured_flag_file, "Setting HBase ACL configured to True for enrichment")
def set_geo_configured(self):
- Logger.info("Setting GEO Configured to True for enrichment")
- File(self.__params.enrichment_geo_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
+ metron_service.set_configured(self.__params.metron_user, self.__params.enrichment_geo_configured_flag_file, "Setting GEO configured to True for enrichment")
def init_geo(self):
Logger.info("Creating HDFS location for GeoIP database")
@@ -202,7 +183,7 @@ class EnrichmentCommands:
self.__params.hbase_principal_name,
execute_user=self.__params.hbase_user)
cmd = "echo \"create '{0}','{1}'\" | hbase shell -n"
- add_enrichment_cmd = cmd.format(self.__params.enrichment_table, self.__params.enrichment_cf)
+ add_enrichment_cmd = cmd.format(self.__params.enrichment_hbase_table, self.__params.enrichment_hbase_cf)
Execute(add_enrichment_cmd,
tries=3,
try_sleep=5,
@@ -211,7 +192,7 @@ class EnrichmentCommands:
user=self.__params.hbase_user
)
- add_threatintel_cmd = cmd.format(self.__params.threatintel_table, self.__params.threatintel_cf)
+ add_threatintel_cmd = cmd.format(self.__params.threatintel_hbase_table, self.__params.threatintel_hbase_cf)
Execute(add_threatintel_cmd,
tries=3,
try_sleep=5,
@@ -231,7 +212,7 @@ class EnrichmentCommands:
self.__params.hbase_principal_name,
execute_user=self.__params.hbase_user)
cmd = "echo \"grant '{0}', 'RW', '{1}'\" | hbase shell -n"
- add_enrichment_acl_cmd = cmd.format(self.__params.metron_user, self.__params.enrichment_table)
+ add_enrichment_acl_cmd = cmd.format(self.__params.metron_user, self.__params.enrichment_hbase_table)
Execute(add_enrichment_acl_cmd,
tries=3,
try_sleep=5,
@@ -240,7 +221,7 @@ class EnrichmentCommands:
user=self.__params.hbase_user
)
- add_threatintel_acl_cmd = cmd.format(self.__params.metron_user, self.__params.threatintel_table)
+ add_threatintel_acl_cmd = cmd.format(self.__params.metron_user, self.__params.threatintel_hbase_table)
Execute(add_threatintel_acl_cmd,
tries=3,
try_sleep=5,
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
index a72f69e..24feb81 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
@@ -27,6 +27,7 @@ import metron_service
import metron_security
class Enrichment(Script):
+
def install(self, env):
from params import params
env.set_params(params)
@@ -43,6 +44,11 @@ class Enrichment(Script):
group=params.metron_group
)
+ if not metron_service.is_zk_configured(params):
+ metron_service.init_zk_config(params)
+ metron_service.set_zk_configured(params)
+ metron_service.refresh_configs(params)
+
Logger.info("Calling security setup")
storm_security_setup(params)
@@ -58,8 +64,6 @@ class Enrichment(Script):
params.metron_principal_name,
execute_user=params.metron_user)
- metron_service.load_global_config(params)
-
if not commands.is_kafka_configured():
commands.init_kafka_topics()
if params.security_enabled and not commands.is_kafka_acl_configured():
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 8dde568..50457d0 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -18,6 +18,7 @@ limitations under the License.
import os
import time
+from datetime import datetime
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute, File
@@ -57,31 +58,26 @@ class IndexingCommands:
def is_hdfs_perm_configured(self):
return self.__hdfs_perm_configured
- def set_configured(self):
- File(self.__params.indexing_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
-
def is_hbase_configured(self):
return self.__hbase_configured
def is_hbase_acl_configured(self):
return self.__hbase_acl_configured
+ def set_configured(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.indexing_configured_flag_file, "Setting Indexing configured to True")
+
def set_hbase_configured(self):
- Logger.info("Setting HBase Configured to True for indexing")
- File(self.__params.indexing_hbase_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
+ metron_service.set_configured(self.__params.metron_user, self.__params.indexing_hbase_configured_flag_file, "Setting HBase configured to True for indexing")
def set_hbase_acl_configured(self):
- Logger.info("Setting HBase ACL Configured to True for indexing")
- File(self.__params.indexing_hbase_acl_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
+ metron_service.set_configured(self.__params.metron_user, self.__params.indexing_hbase_acl_configured_flag_file, "Setting HBase ACL configured to True for indexing")
+
+ def set_acl_configured(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.indexing_acl_configured_flag_file, "Setting Indexing ACL configured to True")
+
+ def set_hdfs_perm_configured(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.indexing_hdfs_perm_configured_flag_file, "Setting HDFS perm configured to True")
def create_hbase_tables(self):
Logger.info("Creating HBase Tables for indexing")
@@ -91,7 +87,7 @@ class IndexingCommands:
self.__params.hbase_principal_name,
execute_user=self.__params.hbase_user)
cmd = "echo \"create '{0}','{1}'\" | hbase shell -n"
- add_update_cmd = cmd.format(self.__params.update_table, self.__params.update_cf)
+ add_update_cmd = cmd.format(self.__params.update_hbase_table, self.__params.update_hbase_cf)
Execute(add_update_cmd,
tries=3,
try_sleep=5,
@@ -111,7 +107,7 @@ class IndexingCommands:
self.__params.hbase_principal_name,
execute_user=self.__params.hbase_user)
cmd = "echo \"grant '{0}', 'RW', '{1}'\" | hbase shell -n"
- add_update_acl_cmd = cmd.format(self.__params.metron_user, self.__params.update_table)
+ add_update_acl_cmd = cmd.format(self.__params.metron_user, self.__params.update_hbase_table)
Execute(add_update_acl_cmd,
tries=3,
try_sleep=5,
@@ -123,18 +119,6 @@ class IndexingCommands:
Logger.info("Done setting HBase ACLs for indexing")
self.set_hbase_acl_configured()
- def set_acl_configured(self):
- File(self.__params.indexing_acl_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
-
- def set_hdfs_perm_configured(self):
- File(self.__params.indexing_hdfs_perm_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
-
def init_kafka_topics(self):
Logger.info('Creating Kafka topics for indexing')
metron_service.init_kafka_topics(self.__params, [self.__indexing_topic])
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index 68e238a..371cab0 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -49,9 +49,12 @@ class Indexing(Script):
group=params.metron_group
)
- commands = IndexingCommands(params)
- metron_service.load_global_config(params)
+ if not metron_service.is_zk_configured(params):
+ metron_service.init_zk_config(params)
+ metron_service.set_zk_configured(params)
+ metron_service.refresh_configs(params)
+ commands = IndexingCommands(params)
if not commands.is_configured():
commands.init_kafka_topics()
commands.init_hdfs_dir()
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
index cf278f2..84dc805 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
@@ -15,25 +15,122 @@ limitations under the License.
"""
import json
+import os
import subprocess
+from datetime import datetime
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Directory, File
from resource_management.core.resources.system import Execute
-from resource_management.core.source import Template
+from resource_management.core.source import InlineTemplate
from resource_management.libraries.functions import format as ambari_format
from resource_management.libraries.functions.get_user_call_output import \
get_user_call_output
from metron_security import kinit
-def init_config():
- Logger.info('Loading config into ZooKeeper')
+def is_zk_configured(params):
+ return os.path.isfile(params.zk_configured_flag_file)
+
+def init_zk_config(params):
+ Logger.info('Loading ALL Metron config into ZooKeeper - this command should ONLY be executed by Ambari on initial install.')
+ Execute(ambari_format(
+ "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PUSH --input_dir {metron_zookeeper_config_path}"),
+ path=ambari_format("{java_home}/bin")
+ )
+
+def set_configured(user, flag_file, log_msg):
+ Logger.info(log_msg)
+ File(flag_file,
+ content="This file created on: " + datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+ owner=user,
+ mode=0755)
+
+def set_zk_configured(params):
+ set_configured(params.metron_user, params.zk_configured_flag_file, "Setting Zookeeper configured to true")
+
+def build_global_config_patch(params, patch_file):
+ # see RFC 6902 at https://tools.ietf.org/html/rfc6902
+ patch_template = """
+ [
+ {
+ "op": "add",
+ "path": "/es.clustername",
+ "value": "{{ es_cluster_name }}"
+ },
+ {
+ "op": "add",
+ "path": "/es.ip",
+ "value": "{{ es_url }}"
+ },
+ {
+ "op": "add",
+ "path": "/es.date.format",
+ "value": "{{es_date_format}}"
+ },
+ {
+ "op": "add",
+ "path": "/parser.error.topic",
+ "value": "{{parser_error_topic}}"
+ },
+ {
+ "op": "add",
+ "path": "/update.hbase.table",
+ "value": "{{update_hbase_table}}"
+ },
+ {
+ "op": "add",
+ "path": "/update.hbase.cf",
+ "value": "{{update_hbase_cf}}"
+ },
+ {
+ "op": "add",
+ "path": "/profiler.client.period.duration",
+ "value": "{{profiler_period_duration}}"
+ },
+ {
+ "op": "add",
+ "path": "/profiler.client.period.duration.units",
+ "value": "{{profiler_period_units}}"
+ }
+ ]
+ """
+ File(patch_file,
+ content=InlineTemplate(patch_template),
+ owner=params.metron_user,
+ group=params.metron_group)
+
+def patch_global_config(params):
+ patch_file = "/tmp/metron-global-config-patch.json"
+ Logger.info("Setup temporary global config JSON patch (formatting per RFC6902): " + patch_file)
+ build_global_config_patch(params, patch_file)
+
+ Logger.info('Patching global config in ZooKeeper')
+ Execute(ambari_format(
+ "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PATCH --config_type GLOBAL --patch_file " + patch_file),
+ path=ambari_format("{java_home}/bin")
+ )
+
+def pull_config(params):
+ Logger.info('Pulling all Metron configs down from ZooKeeper to local file system')
+ Logger.info('NOTE - THIS IS OVERWRITING THE LOCAL METRON CONFIG DIR WITH ZOOKEEPER CONTENTS: ' + params.metron_zookeeper_config_path)
Execute(ambari_format(
- "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path} -z {zookeeper_quorum}"),
+ "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PULL --output_dir {metron_zookeeper_config_path} --force"),
path=ambari_format("{java_home}/bin")
)
+# pushes json patches to zookeeper based on Ambari parameters that are configurable by the user
+def refresh_configs(params):
+ if not is_zk_configured(params):
+ Logger.warning("The expected flag file '" + params.zk_configured_flag_file + "'indicating that Zookeeper has been configured does not exist. Skipping patching. An administrator should look into this.")
+ return
+
+ Logger.info("Patch global config in Zookeeper")
+ patch_global_config(params)
+ Logger.info("Done patching global config")
+
+ Logger.info("Pull zookeeper config locally")
+ pull_config(params)
def get_running_topologies(params):
Logger.info('Getting Running Storm Topologies from Storm REST Server')
@@ -76,27 +173,6 @@ def get_running_topologies(params):
Logger.info("Topologies: " + str(topologiesDict))
return topologiesDict
-
-def load_global_config(params):
- Logger.info('Create Metron Local Config Directory')
- Logger.info("Configure Metron global.json")
-
- directories = [params.metron_zookeeper_config_path]
- Directory(directories,
- mode=0755,
- owner=params.metron_user,
- group=params.metron_group
- )
-
- File(ambari_format("{metron_zookeeper_config_path}/global.json"),
- content=Template("global.json.j2"),
- owner=params.metron_user,
- group=params.metron_group
- )
-
- init_config()
-
-
def init_kafka_topics(params, topics):
Logger.info('Creating Kafka topics')
@@ -125,7 +201,6 @@ def init_kafka_topics(params, topics):
user=params.kafka_user, tries=3, try_sleep=5, logoutput=True)
Logger.info("Done creating Kafka topics")
-
def init_kafka_acls(params, topics, groups):
Logger.info('Creating Kafka ACLs')
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index b273bd6..3bfe8e3 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -60,6 +60,8 @@ metron_escalation_topic = config['configurations']['metron-rest-env']['metron_es
metron_config_path = metron_home + '/config'
metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir
metron_zookeeper_config_path = status_params.metron_zookeeper_config_path
+# indicates if zk_load_configs.sh --mode PUSH has been executed
+zk_configured_flag_file = status_params.zk_configured_flag_file
parsers_configured_flag_file = status_params.parsers_configured_flag_file
parsers_acl_configured_flag_file = status_params.parsers_acl_configured_flag_file
rest_acl_configured_flag_file = status_params.rest_acl_configured_flag_file
@@ -73,7 +75,6 @@ indexing_acl_configured_flag_file = status_params.indexing_acl_configured_flag_f
indexing_hbase_configured_flag_file = status_params.indexing_hbase_configured_flag_file
indexing_hbase_acl_configured_flag_file = status_params.indexing_hbase_acl_configured_flag_file
indexing_hdfs_perm_configured_flag_file = status_params.indexing_hdfs_perm_configured_flag_file
-global_json_template = config['configurations']['metron-env']['global-json']
global_properties_template = config['configurations']['metron-env']['elasticsearch-properties']
# Elasticsearch hosts and port management
@@ -170,14 +171,14 @@ HdfsResource = functools.partial(
# HBase
enrichment_hbase_provider_impl = 'org.apache.metron.hbase.HTableProvider'
-enrichment_table = status_params.enrichment_table
-enrichment_cf = status_params.enrichment_cf
-update_table = status_params.update_table
-update_cf = status_params.update_cf
+enrichment_hbase_table = status_params.enrichment_hbase_table
+enrichment_hbase_cf = status_params.enrichment_hbase_cf
+update_hbase_table = status_params.update_hbase_table
+update_hbase_cf = status_params.update_hbase_cf
-threatintel_table = status_params.threatintel_table
-threatintel_cf = status_params.threatintel_cf
+threatintel_hbase_table = status_params.threatintel_hbase_table
+threatintel_hbase_cf = status_params.threatintel_hbase_cf
# Kafka Topics
ambari_kafka_service_check_topic = 'ambari_kafka_service_check'
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 1935f2f..9ff1430 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -26,12 +26,14 @@ from resource_management.libraries.functions.version import format_stack_version
config = Script.get_config()
metron_user = config['configurations']['metron-env']['metron_user']
-
-# Parsers
-parsers = config['configurations']['metron-parsers-env']['parsers']
metron_home = config['configurations']['metron-env']['metron_home']
metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir']
metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}')
+# indicates if zk_load_configs.sh --mode PUSH has been executed
+zk_configured_flag_file = metron_zookeeper_config_path + '/../metron_zookeeper_configured'
+
+# Parsers
+parsers = config['configurations']['metron-parsers-env']['parsers']
parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured'
parsers_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_acl_configured'
rest_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_rest_acl_configured'
@@ -45,12 +47,12 @@ enrichment_hbase_configured_flag_file = metron_zookeeper_config_path + '/../metr
enrichment_hbase_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_acl_configured'
enrichment_geo_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_geo_configured'
-enrichment_table = 'enrichment'
-enrichment_cf = 't'
-threatintel_table = 'threatintel'
-threatintel_cf = 't'
-update_table = 'metron_update'
-update_cf = 't'
+enrichment_hbase_table = config['configurations']['metron-enrichment-env']['enrichment_hbase_table']
+enrichment_hbase_cf = config['configurations']['metron-enrichment-env']['enrichment_hbase_cf']
+threatintel_hbase_table = config['configurations']['metron-enrichment-env']['threatintel_hbase_table']
+threatintel_hbase_cf = config['configurations']['metron-enrichment-env']['threatintel_hbase_cf']
+update_hbase_table = config['configurations']['metron-indexing-env']['update_hbase_table']
+update_hbase_cf = config['configurations']['metron-indexing-env']['update_hbase_cf']
# Profiler
metron_profiler_topology = 'profiler'
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
index ccad5d5..9483498 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
@@ -23,6 +23,7 @@ import re
import subprocess
import time
+from datetime import datetime
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute, File
@@ -56,16 +57,10 @@ class ParserCommands:
return self.__acl_configured
def set_configured(self):
- File(self.__params.parsers_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
+ metron_service.set_configured(self.__params.metron_user, self.__params.parsers_configured_flag_file, "Setting Parsers configured to true")
def set_acl_configured(self):
- File(self.__params.parsers_acl_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
+ metron_service.set_configured(self.__params.metron_user, self.__params.parsers_acl_configured_flag_file, "Setting Parsers ACL configured to true")
def init_parsers(self):
Logger.info(
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_master.py
index 9d3ebde..d5b333a 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_master.py
@@ -40,7 +40,11 @@ class ParserMaster(Script):
def configure(self, env, upgrade_type=None, config_dir=None):
from params import params
env.set_params(params)
- metron_service.load_global_config(params)
+
+ if not metron_service.is_zk_configured(params):
+ metron_service.init_zk_config(params)
+ metron_service.set_zk_configured(params)
+ metron_service.refresh_configs(params)
commands = ParserCommands(params)
if not commands.is_configured():
commands.init_parsers()
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
index 5c31896..21c1225 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
@@ -18,6 +18,7 @@ limitations under the License.
import os
import time
+from datetime import datetime
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute, File
@@ -52,31 +53,23 @@ class ProfilerCommands:
def is_acl_configured(self):
return self.__acl_configured
- def set_configured(self):
- File(self.__params.profiler_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
-
def is_hbase_configured(self):
return self.__hbase_configured
def is_hbase_acl_configured(self):
return self.__hbase_acl_configured
+ def set_configured(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.profiler_configured_flag_file, "Setting Profiler configured flag to true")
+
+ def set_acl_configured(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.profiler_acl_configured_flag_file, "Setting Profiler acl configured flag to true")
+
def set_hbase_configured(self):
- Logger.info("Setting HBase Configured to True for profiler")
- File(self.__params.profiler_hbase_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
+ metron_service.set_configured(self.__params.metron_user, self.__params.profiler_hbase_configured_flag_file, "Setting HBase configured to True for profiler")
def set_hbase_acl_configured(self):
- Logger.info("Setting HBase ACL Configured to True for profiler")
- File(self.__params.profiler_hbase_acl_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
+ metron_service.set_configured(self.__params.metron_user, self.__params.profiler_hbase_acl_configured_flag_file, "Setting HBase ACL configured to True for profiler")
def create_hbase_tables(self):
Logger.info("Creating HBase Tables for profiler")
@@ -122,12 +115,6 @@ class ProfilerCommands:
Logger.info("Done setting HBase ACLs for profiler")
self.set_hbase_acl_configured()
- def set_acl_configured(self):
- File(self.__params.profiler_acl_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
-
def start_profiler_topology(self, env):
Logger.info('Starting ' + self.__profiler_topology)
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py
index 77c32f0..ddfadd2 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py
@@ -50,12 +50,12 @@ class Profiler(Script):
group=params.metron_group
)
- commands = ProfilerCommands(params)
- metron_service.load_global_config(params)
-
- if not commands.is_configured():
- commands.set_configured()
+ if not metron_service.is_zk_configured(params):
+ metron_service.init_zk_config(params)
+ metron_service.set_zk_configured(params)
+ metron_service.refresh_configs(params)
+ commands = ProfilerCommands(params)
if not commands.is_hbase_configured():
commands.create_hbase_tables()
if params.security_enabled and not commands.is_hbase_acl_configured():
@@ -66,6 +66,8 @@ class Profiler(Script):
Logger.info("Calling security setup")
storm_security_setup(params)
+ if not commands.is_configured():
+ commands.set_configured()
def start(self, env, upgrade_type=None):
from params import params
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
index cf29a28..9424075 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@ -19,6 +19,7 @@ limitations under the License.
"""
import os
+from datetime import datetime
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute, File
from resource_management.libraries.functions.format import format
@@ -40,15 +41,12 @@ class RestCommands:
return self.__acl_configured
def set_acl_configured(self):
- File(self.__params.rest_acl_configured_flag_file,
- content="",
- owner=self.__params.metron_user,
- mode=0755)
+ metron_service.set_configured(self.__params.metron_user, self.__params.rest_acl_configured_flag_file, "Setting REST ACL configured to true")
def init_kafka_topics(self):
- Logger.info('Creating Kafka topics for rest')
- topics = [self.__params.metron_escalation_topic]
- metron_service.init_kafka_topics(self.__params, topics)
+ Logger.info('Creating Kafka topics for rest')
+ topics = [self.__params.metron_escalation_topic]
+ metron_service.init_kafka_topics(self.__params, topics)
def init_kafka_acls(self):
Logger.info('Creating Kafka ACLs for rest')
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2
deleted file mode 100644
index 67226ff..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/global.json.j2
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "es.clustername": "{{ es_cluster_name }}",
- "es.ip": "{{ es_url }}",
- "es.date.format": "{{es_date_format}}",
- "parser.error.topic": "{{parser_error_topic}}",
- "update.hbase.table": "{{update_table}}",
- "update.hbase.cf": "{{update_cf}}"
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index 207051d..117aff6 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -464,11 +464,11 @@
"subsection-name": "subsection-indexing-kafka"
},
{
- "config": "metron-indexing-env/update_table",
+ "config": "metron-indexing-env/update_hbase_table",
"subsection-name": "subsection-indexing-update"
},
{
- "config": "metron-indexing-env/update_cf",
+ "config": "metron-indexing-env/update_hbase_cf",
"subsection-name": "subsection-indexing-update"
},
{
@@ -810,13 +810,13 @@
}
},
{
- "config": "metron-indexing-env/update_table",
+ "config": "metron-indexing-env/update_hbase_table",
"widget": {
"type": "text-field"
}
},
{
- "config": "metron-indexing-env/update_cf",
+ "config": "metron-indexing-env/update_hbase_cf",
"widget": {
"type": "text-field"
}
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 6065d57..26f626b 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -114,9 +114,12 @@ This package installs the Metron common files %{metron_home}
%dir %{metron_root}
%dir %{metron_home}
%dir %{metron_home}/bin
+%dir %{metron_home}/config
+%dir %{metron_home}/config/zookeeper
%dir %{metron_home}/lib
%{metron_home}/bin/zk_load_configs.sh
%{metron_home}/bin/stellar
+%{metron_home}/config/zookeeper/global.json
%attr(0644,root,root) %{metron_home}/lib/metron-common-%{full_version}.jar
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java
index 59d5957..824fb4b 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java
@@ -17,6 +17,17 @@
*/
package org.apache.metron.rest.service.impl;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.Map;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.DeleteBuilder;
import org.apache.curator.framework.api.GetDataBuilder;
@@ -31,18 +42,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
@SuppressWarnings("ALL")
public class GlobalConfigServiceImplTest {
@Rule
@@ -134,7 +133,7 @@ public class GlobalConfigServiceImplTest {
exception.expect(RestException.class);
SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
- when(setDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot(), "{}".getBytes())).thenThrow(Exception.class);
+ when(setDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot(), "{ }".getBytes())).thenThrow(Exception.class);
when(curatorFramework.setData()).thenReturn(setDataBuilder);
@@ -144,11 +143,11 @@ public class GlobalConfigServiceImplTest {
@Test
public void saveShouldReturnSameConfigThatIsPassedOnSuccessfulSave() throws Exception {
SetDataBuilder setDataBuilder = mock(SetDataBuilder.class);
- when(setDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot(), "{}".getBytes())).thenReturn(new Stat());
+ when(setDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot(), "{ }".getBytes())).thenReturn(new Stat());
when(curatorFramework.setData()).thenReturn(setDataBuilder);
assertEquals(new HashMap<>(), globalConfigService.save(new HashMap<>()));
- verify(setDataBuilder).forPath(eq(ConfigurationType.GLOBAL.getZookeeperRoot()), eq("{}".getBytes()));
+ verify(setDataBuilder).forPath(eq(ConfigurationType.GLOBAL.getZookeeperRoot()), eq("{ }".getBytes()));
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/README.md b/metron-platform/metron-common/README.md
index fbc3c10..54738f8 100644
--- a/metron-platform/metron-common/README.md
+++ b/metron-platform/metron-common/README.md
@@ -98,16 +98,28 @@ utility program to assist in this called `$METRON_HOME/bin/zk_load_config.sh`
This has the following options:
```
+ -c,--config_type <CONFIG_TYPE> The configuration type: GLOBAL,
+ PARSER, ENRICHMENT, INDEXING,
+ PROFILER
-f,--force Force operation
-h,--help Generate Help screen
-i,--input_dir <DIR> The input directory containing
the configuration files named
like "$source.json"
-m,--mode <MODE> The mode of operation: DUMP,
- PULL, PUSH
+ PULL, PUSH, PATCH
+ -n,--config_name <CONFIG_NAME> The configuration name: bro,
+ yaf, snort, squid, etc.
-o,--output_dir <DIR> The output directory which will
store the JSON configuration
from Zookeeper
+ -pk,--patch_key <PATCH_KEY> The key to modify
+ -pm,--patch_mode <PATCH_MODE> One of: ADD, REMOVE - relevant
+ only for key/value patches,
+ i.e. when a patch file is not
+ used.
+ -pf,--patch_file <PATCH_FILE> Path to the patch file.
+ -pv,--patch_value <PATCH_VALUE> Value to use in the patch.
-z,--zk_quorum <host:port,[host:port]*> Zookeeper Quorum URL
(zk1:port,zk2:port,...)
```
@@ -115,8 +127,150 @@ This has the following options:
Usage examples:
* To dump the existing configs from zookeeper on the singlenode vagrant machine: `$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m DUMP`
+* To dump the existing GLOBAL configs from zookeeper on the singlenode vagrant machine: `$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m DUMP -c GLOBAL`
* To push the configs into zookeeper on the singlenode vagrant machine: `$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PUSH -i $METRON_HOME/config/zookeeper`
-* To pull the configs from zookeeper to the singlenode vagrant machine disk: `$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PULL -o $METRON_HOME/config/zookeeper -f`
+* To push only the GLOBAL configs into zookeeper on the singlenode vagrant machine: `$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PUSH -i $METRON_HOME/config/zookeeper -c GLOBAL`
+* To push only the PARSER configs into zookeeper on the singlenode vagrant machine: `$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PUSH -i $METRON_HOME/config/zookeeper -c PARSER`
+* To push only the PARSER 'bro' configs into zookeeper on the singlenode vagrant machine: `$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PUSH -i $METRON_HOME/config/zookeeper -c PARSER -n bro`
+* To pull all configs from zookeeper to the singlenode vagrant machine disk: `$METRON_HOME/bin/zk_load_configs.sh -z node1:2181 -m PULL -o $METRON_HOME/config/zookeeper -f`
+
+## Patching mechanism
+
+The configuration management utility leverages a JSON patching library that conforms to [RFC-6902 spec](https://tools.ietf.org/html/rfc6902). We're using the zjsonpatch library implementation from here - https://github.com/flipkart-incubator/zjsonpatch.
+There are a couple options for leveraging patching. You can choose to patch the Zookeeper config via patch file:
+
+`$METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pf /tmp/mypatch.txt`
+
+or key/value pair:
+
+`$METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pm ADD -pk foo -pv \"\"bar\"\"`
+
+The options exposed via patch file are the full range of options from RFC-6902:
+ - ADD
+ - REMOVE
+ - REPLACE
+ - MOVE
+ - COPY
+ - TEST
+
+whereas with key/value patching, we only current expose ADD and REMOVE. Note that ADD will function as a REPLACE when the key already exists.
+
+### Patch File
+
+Let's say we want to add a complex JSON object to our configuration with a patch file. e.g.
+```
+"foo" : {
+ "bar" : {
+ "baz" : [ "bazval1", "bazval2" ]
+ }
+ }
+```
+
+We would write a patch file "/tmp/mypatch.txt" with contents:
+```
+[
+ {
+ "op": "add",
+ "path": "/foo",
+ "value": { "bar" : { "baz" : [ "bazval1", "bazval2" ] } }
+ }
+]
+```
+
+And submit via zk_load_configs as follows:
+```
+ $METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pf /tmp/mypatch.txt
+```
+
+### Patch Key/Value
+
+Now let's try the same without using a patch file, instead using the patch_key and patch_value options right from the command line utility. This would like like the following.
+
+```
+$METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pm ADD -pk "/foo" -pv "{ \"bar\" : { \"baz\" : [ \"bazval1\", \"bazval2\" ] } }"
+```
+
+### Applying Multiple Patches
+
+Applying multiple patches is also pretty straightforward. You can achieve this in a single command using patch files, or simply execute multiple commands in sequence using the patch_key/value approach.
+
+Let's say we wanted to add the following to our global config:
+```
+"apache" : "metron",
+"is" : "the best",
+"streaming" : "analytics platform"
+```
+
+and remove the /foo key from the previous example.
+
+Create a patch file /tmp/mypatch.txt with four separate patch operations.
+```
+[
+ {
+ "op": "remove",
+ "path": "/foo"
+ },
+ {
+ "op": "add",
+ "path": "/apache",
+ "value": "metron"
+ },
+ {
+ "op": "add",
+ "path": "/is",
+ "value": "the best"
+ },
+ {
+ "op": "add",
+ "path": "/streaming",
+ "value": "analytics platform"
+ }
+]
+```
+
+Now submit again and you should see a Global config with the "foo" key removed and three new keys added.
+```
+ $METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c GLOBAL -pf /tmp/mypatch.txt
+```
+
+### Notes On Patching
+
+For any given patch key, the last/leaf node in the key's parent *must* exist, otherwise an exception will be thrown. For example, if you want to add the following:
+```
+"foo": {
+ "bar": "baz"
+}
+```
+
+It is not sufficient to use /foo/bar as a key if foo does not already exist. You would either need to incrementally build the JSON and make this a two step process
+```
+[
+ {
+ "op": "add",
+ "path": "/foo",
+ "value": { }
+ },
+ {
+ "op": "add",
+ "path": "/foo/bar",
+ "value": "baz"
+ }
+]
+```
+
+Or provide the value as a complete JSON object.
+```
+[
+ {
+ "op": "add",
+ "path": "/foo",
+ "value": { "bar" : "baz" }
+ }
+]
+```
+
+The REMOVE operation is idempotent. Running the remove command on the same key multiple
+times will not fail once the key has been removed.
# Topology Errors
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index 9356e13..3054881 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -352,6 +352,25 @@
<artifactId>stream</artifactId>
<version>2.9.5</version>
</dependency>
+ <dependency>
+ <groupId>com.flipkart.zjsonpatch</groupId>
+ <artifactId>zjsonpatch</artifactId>
+ <version>0.3.4</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/assembly/assembly.xml b/metron-platform/metron-common/src/main/assembly/assembly.xml
index 3595284..e0e4f14 100644
--- a/metron-platform/metron-common/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-common/src/main/assembly/assembly.xml
@@ -19,6 +19,18 @@
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
+ <directory>${project.basedir}/src/main/config</directory>
+ <outputDirectory>config</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0644</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
<directory>${project.basedir}/src/main/scripts</directory>
<outputDirectory>bin</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/config/zookeeper/global.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/config/zookeeper/global.json b/metron-platform/metron-common/src/main/config/zookeeper/global.json
new file mode 100644
index 0000000..dc7e71f
--- /dev/null
+++ b/metron-platform/metron-common/src/main/config/zookeeper/global.json
@@ -0,0 +1,8 @@
+{
+ "es.clustername": "metron",
+ "es.ip": "node1:9300",
+ "es.date.format": "yyyy.MM.dd.HH",
+ "parser.error.topic": "indexing",
+ "update.hbase.table": "metron_update",
+ "update.hbase.cf": "t"
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/cli/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/cli/ConfigurationManager.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/cli/ConfigurationManager.java
index 9685841..8618179 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/cli/ConfigurationManager.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/cli/ConfigurationManager.java
@@ -21,49 +21,104 @@ package org.apache.metron.common.cli;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.io.Files;
-import org.apache.commons.cli.*;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
-import org.apache.metron.common.configuration.ConfigurationType;
-
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.PrintWriter;
+import java.lang.invoke.MethodHandles;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
+import java.util.Optional;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.metron.common.configuration.ConfigurationType;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ConfigurationManager {
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
public enum ConfigurationOptions {
- HELP("h", s -> new Option(s, "help", false, "Generate Help screen"))
- ,INPUT("i", s -> OptionBuilder.isRequired(false)
- .withLongOpt("input_dir")
- .hasArg()
- .withArgName("DIR")
- .withDescription("The input directory containing the configuration files named like \"$source.json\"")
- .create(s)
- )
- ,OUTPUT("o", s -> OptionBuilder.isRequired(false)
- .hasArg()
- .withLongOpt("output_dir")
- .withArgName("DIR")
- .withDescription("The output directory which will store the JSON configuration from Zookeeper")
- .create(s)
- )
- ,ZK_QUORUM("z", s -> OptionBuilder.isRequired(true)
- .hasArg()
- .withLongOpt("zk_quorum")
- .withArgName("host:port,[host:port]*")
- .withDescription("Zookeeper Quorum URL (zk1:port,zk2:port,...)")
- .create(s)
- )
- ,MODE("m", s -> OptionBuilder.isRequired(true)
- .hasArg()
- .withLongOpt("mode")
- .withArgName("MODE")
- .withDescription("The mode of operation: DUMP, PULL, PUSH")
- .create(s)
- )
- ,FORCE("f", s -> new Option(s, "force", false, "Force operation"))
+ HELP("h", s -> new Option(s, "help", false, "Generate Help screen")),
+ INPUT("i", s -> OptionBuilder.isRequired(false)
+ .hasArg()
+ .withLongOpt("input_dir")
+ .withArgName("DIR")
+ .withDescription("The input directory containing the configuration files named like \"$source.json\"")
+ .create(s)
+ ),
+ OUTPUT("o", s -> OptionBuilder.isRequired(false)
+ .hasArg()
+ .withLongOpt("output_dir")
+ .withArgName("DIR")
+ .withDescription("The output directory which will store the JSON configuration from Zookeeper")
+ .create(s)
+ ),
+ ZK_QUORUM("z", s -> OptionBuilder.isRequired(true)
+ .hasArg()
+ .withLongOpt("zk_quorum")
+ .withArgName("host:port,[host:port]*")
+ .withDescription("Zookeeper Quorum URL (zk1:port,zk2:port,...)")
+ .create(s)
+ ),
+ MODE("m", s -> OptionBuilder.isRequired(true)
+ .hasArg()
+ .withLongOpt("mode")
+ .withArgName("MODE")
+ .withDescription("The mode of operation: DUMP, PULL, PUSH, PATCH")
+ .create(s)
+ ),
+ CONFIG_TYPE("c", s -> OptionBuilder.isRequired(false)
+ .hasArg()
+ .withLongOpt("config_type")
+ .withArgName("CONFIG_TYPE")
+ .withDescription("The configuration type: GLOBAL, PARSER, ENRICHMENT, INDEXING, PROFILER")
+ .create(s)
+ ),
+ CONFIG_NAME("n", s -> OptionBuilder.isRequired(false)
+ .hasArg()
+ .withLongOpt("config_name")
+ .withArgName("CONFIG_NAME")
+ .withDescription("The configuration name: bro, yaf, snort, squid, etc.")
+ .create(s)
+ ),
+ PATCH_FILE("pf", s -> OptionBuilder.isRequired(false)
+ .hasArg()
+ .withLongOpt("patch_file")
+ .withArgName("PATCH_FILE")
+ .withDescription("Path to the patch file.")
+ .create(s)
+ ),
+ PATCH_MODE("pm", s -> OptionBuilder.isRequired(false)
+ .hasArg()
+ .withLongOpt("patch_mode")
+ .withArgName("PATCH_MODE")
+ .withDescription("One of: ADD, REMOVE - relevant only for key/value patches, i.e. when a patch file is not used.")
+ .create(s)
+ ),
+ PATCH_KEY("pk", s -> OptionBuilder.isRequired(false)
+ .hasArg()
+ .withLongOpt("patch_key")
+ .withArgName("PATCH_KEY")
+ .withDescription("The key to modify")
+ .create(s)
+ ),
+ PATCH_VALUE("pv", s -> OptionBuilder.isRequired(false)
+ .hasArg()
+ .withLongOpt("patch_value")
+ .withArgName("PATCH_VALUE")
+ .withDescription("Value to use in the patch.")
+ .create(s)
+ ),
+ FORCE("f", s -> new Option(s, "force", false, "Force operation"))
;
Option option;
String shortCode;
@@ -111,11 +166,28 @@ public class ConfigurationManager {
}
}
+ public enum PatchMode {
+ ADD, REMOVE, REPLACE, MOVE, COPY, TEST;
+ }
+
+ /**
+ * Dumps all config
+ * @param client
+ * @throws Exception
+ */
public void dump(CuratorFramework client) throws Exception {
ConfigurationsUtils.dumpConfigs(System.out, client);
}
-
+ /**
+ * Dumps specific config type only
+ * @param client
+ * @param type
+ * @throws Exception
+ */
+ public void dump(CuratorFramework client, ConfigurationType type, Optional<String> configName) throws Exception {
+ ConfigurationsUtils.dumpConfigs(System.out, client, type, configName);
+ }
public void pull(CuratorFramework client, String outFileStr, final boolean force) throws Exception {
final File outputDir = new File(outFileStr);
@@ -130,7 +202,7 @@ public class ConfigurationManager {
public void visit(ConfigurationType configurationType, String name, String data) {
File out = getFile(outputDir, configurationType, name);
if (!out.exists() || force) {
- if(!out.exists()) {
+ if (!out.exists()) {
out.getParentFile().mkdirs();
}
try {
@@ -138,8 +210,7 @@ public class ConfigurationManager {
} catch (IOException e) {
throw new RuntimeException("Sorry, something bad happened writing the config to " + out.getAbsolutePath() + ": " + e.getMessage(), e);
}
- }
- else if(out.exists() && !force) {
+ } else if (out.exists() && !force) {
throw new IllegalStateException("Unable to overwrite existing file (" + out.getAbsolutePath() + ") without the force flag (-f or --force) being set.");
}
}
@@ -147,12 +218,23 @@ public class ConfigurationManager {
}
public void push(String inputDirStr, CuratorFramework client) throws Exception {
- final File inputDir = new File(inputDirStr);
+ final File inputDir = new File(inputDirStr);
- if(!inputDir.exists() || !inputDir.isDirectory()) {
- throw new IllegalStateException("Input directory: " + inputDir + " does not exist or is not a directory.");
- }
- ConfigurationsUtils.uploadConfigsToZookeeper(inputDirStr, client);
+ if (!inputDir.exists() || !inputDir.isDirectory()) {
+ throw new IllegalStateException("Input directory: " + inputDir + " does not exist or is not a directory.");
+ }
+ ConfigurationsUtils.uploadConfigsToZookeeper(inputDirStr, client);
+ }
+
+ public void push(String inputDirStr, CuratorFramework client, ConfigurationType type, Optional<String> configName)
+ throws Exception {
+ final File inputDir = new File(inputDirStr);
+
+ if (!inputDir.exists() || !inputDir.isDirectory()) {
+ throw new IllegalStateException(
+ "Input directory: " + inputDir + " does not exist or is not a directory.");
+ }
+ ConfigurationsUtils.uploadConfigsToZookeeper(inputDirStr, client, type, configName);
}
public void run(CommandLine cli) throws Exception {
@@ -164,27 +246,101 @@ public class ConfigurationManager {
public void run(CuratorFramework client, CommandLine cli) throws Exception {
final boolean force = ConfigurationOptions.FORCE.has(cli);
String mode = ConfigurationOptions.MODE.get(cli);
+ Optional<String> configType = Optional.ofNullable(ConfigurationOptions.CONFIG_TYPE.get(cli));
+ Optional<String> configName = Optional.ofNullable(ConfigurationOptions.CONFIG_NAME.get(cli));
- if (mode.toLowerCase().equals("push")) {
- String inputDirStr = ConfigurationOptions.INPUT.get(cli);
- push(inputDirStr, client);
- }
- else {
+ switch (mode.toLowerCase()) {
- switch (mode.toLowerCase()) {
+ case "push":
+ String inputDirStr = ConfigurationOptions.INPUT.get(cli);
+ if (StringUtils.isEmpty(inputDirStr)) {
+ throw new IllegalArgumentException("Input directory is required when performing a PUSH operation.");
+ }
+ if (configType.isPresent()) {
+ push(inputDirStr, client, ConfigurationType.valueOf(configType.get()), configName);
+ } else {
+ push(inputDirStr, client);
+ }
- case "dump":
+ case "dump":
+ if (configType.isPresent()) {
+ dump(client, ConfigurationType.valueOf(configType.get()), configName);
+ } else {
dump(client);
- break;
+ }
+ break;
- case "pull":
- pull(client, ConfigurationOptions.OUTPUT.get(cli), force);
- break;
+ case "pull":
+ pull(client, ConfigurationOptions.OUTPUT.get(cli), force);
+ break;
+
+ case "patch":
+ if(configType.isPresent()) {
+ Optional<String> patchPath = Optional.ofNullable(ConfigurationOptions.PATCH_FILE.get(cli));
+ Optional<String> patchMode = Optional.ofNullable(ConfigurationOptions.PATCH_MODE.get(cli));
+ Optional<String> patchKey = Optional.ofNullable(ConfigurationOptions.PATCH_KEY.get(cli));
+ Optional<String> patchValue = Optional.ofNullable(ConfigurationOptions.PATCH_VALUE.get(cli));
+ patch(client, ConfigurationType.valueOf(configType.get()), configName, patchMode, patchPath, patchKey, patchValue);
+ } else {
+ throw new IllegalArgumentException("Patch requires config type");
+ }
+ break;
+
+ default:
+ throw new IllegalStateException("Invalid mode: " + mode + " expected DUMP, PULL, PUSH, or PATCH");
+ }
- default:
- throw new IllegalStateException("Invalid mode: " + mode + " expected DUMP, PULL or PUSH");
+ }
+
+ private void patch(CuratorFramework client, ConfigurationType configType,
+ Optional<String> configName, Optional<String> patchMode, Optional<String> patchPath,
+ Optional<String> patchKey, Optional<String> patchValue) {
+ try {
+ byte[] patchData = null;
+ if (patchKey.isPresent()) {
+ patchData = buildPatch(patchMode, patchKey, patchValue).getBytes(StandardCharsets.UTF_8);
+ } else {
+ patchData = java.nio.file.Files.readAllBytes(Paths.get(patchPath.get()));
}
+ ConfigurationsUtils.applyConfigPatchToZookeeper(configType, configName, patchData, client);
+ } catch (IOException e) {
+ LOG.error("Unable to load patch file '%s'", patchPath, e);
+ } catch (Exception e) {
+ LOG.error("Unable to apply patch to Zookeeper config", e);
+ }
+ }
+
+ private String buildPatch(Optional<String> patchMode, Optional<String> patchKey,
+ Optional<String> patchValue) {
+ PatchMode mode = PatchMode.ADD;
+ if (patchMode.isPresent()) {
+ mode = PatchMode.valueOf(patchMode.get());
+ }
+ String patch = "";
+ switch (mode) {
+ case ADD:
+ if (!patchKey.isPresent() || !patchValue.isPresent()) {
+ throw new IllegalArgumentException(
+ "Key and value are required to apply patches without a file");
+ }
+ patch = String.format("[ { \"op\": \"%s\", \"path\": \"%s\", \"value\": %s } ]",
+ patchMode.get().toString().toLowerCase(),
+ patchKey.get(),
+ patchValue.get());
+ break;
+ case REMOVE:
+ if (!patchKey.isPresent()) {
+ throw new IllegalArgumentException(
+ "Key is required to apply a remove patch without a file");
+ }
+ patch = String.format("[ { \"op\": \"%s\", \"path\": \"%s\" } ]",
+ patchMode.get().toString().toLowerCase(),
+ patchKey.get());
+ break;
+ default:
+ throw new UnsupportedOperationException("Patch mode not supported: " + mode.toString());
}
+ return patch;
}
private static File getFile(File baseDir, ConfigurationType configurationType, String name) {
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationType.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationType.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationType.java
index 23e91c8..e803341 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationType.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationType.java
@@ -69,20 +69,20 @@ public enum ConfigurationType implements Function<String, Object> {
}
});
- String name;
+ String typeName;
String directory;
String zookeeperRoot;
Function<String,?> deserializer;
- ConfigurationType(String name, String directory, Function<String, ?> deserializer) {
- this.name = name;
+ ConfigurationType(String typeName, String directory, Function<String, ?> deserializer) {
+ this.typeName = typeName;
this.directory = directory;
- this.zookeeperRoot = Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + name;
+ this.zookeeperRoot = Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + typeName;
this.deserializer = deserializer;
}
- public String getName() {
- return name;
+ public String getTypeName() {
+ return typeName;
}
public String getDirectory() {
@@ -101,4 +101,5 @@ public enum ConfigurationType implements Function<String, Object> {
public String getZookeeperRoot() {
return zookeeperRoot;
}
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java
index 2d03f30..6c518b1 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java
@@ -40,7 +40,7 @@ public class Configurations implements Serializable {
@SuppressWarnings("unchecked")
public Map<String, Object> getGlobalConfig() {
- return (Map<String, Object>) configurations.getOrDefault(ConfigurationType.GLOBAL.getName(), new HashMap());
+ return (Map<String, Object>) configurations.getOrDefault(ConfigurationType.GLOBAL.getTypeName(), new HashMap());
}
public List<FieldValidator> getFieldValidations() {
@@ -59,7 +59,7 @@ public class Configurations implements Serializable {
}
public void updateGlobalConfig(Map<String, Object> globalConfig) {
- configurations.put(ConfigurationType.GLOBAL.getName(), globalConfig);
+ configurations.put(ConfigurationType.GLOBAL.getTypeName(), globalConfig);
validations = FieldValidator.readValidations(getGlobalConfig());
}