You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2017/07/12 13:20:24 UTC
[2/2] metron git commit: METRON-1017 Ambari components should be
separate (merrimanr) closes apache/metron#634
METRON-1017 Ambari components should be separate (merrimanr) closes apache/metron#634
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a9b01d1e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a9b01d1e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a9b01d1e
Branch: refs/heads/master
Commit: a9b01d1e373091662f4dff3ac3acf7cec181e103
Parents: cf165ff
Author: merrimanr <me...@gmail.com>
Authored: Wed Jul 12 08:20:03 2017 -0500
Committer: merrimanr <me...@apache.org>
Committed: Wed Jul 12 08:20:03 2017 -0500
----------------------------------------------------------------------
.../configuration/metron-enrichment-env.xml | 174 ++++++++
.../METRON/CURRENT/configuration/metron-env.xml | 413 +------------------
.../configuration/metron-indexing-env.xml | 165 ++++++++
.../configuration/metron-management-ui-env.xml | 27 ++
.../configuration/metron-parsers-env.xml | 33 ++
.../CURRENT/configuration/metron-rest-env.xml | 112 +++++
.../common-services/METRON/CURRENT/metainfo.xml | 16 +
.../package/scripts/enrichment_commands.py | 6 +-
.../package/scripts/indexing_commands.py | 24 +-
.../package/scripts/params/params_linux.py | 117 +++---
.../package/scripts/params/status_params.py | 19 +-
.../CURRENT/package/scripts/parser_commands.py | 4 +-
.../METRON/CURRENT/quicklinks/quicklinks.json | 4 +-
.../METRON/CURRENT/themes/metron_theme.json | 219 +++++-----
.../roles/ambari_config/vars/single_node_vm.yml | 6 +-
15 files changed, 731 insertions(+), 608 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
new file mode 100644
index 0000000..e042235
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
@@ -0,0 +1,174 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>geoip_url</name>
+ <value>http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz</value>
+ <description>Location of the GeoIP data to load.</description>
+ <display-name>GEOIP Load Datafile URL</display-name>
+ </property>
+ <property>
+ <name>enrichment_host_known_hosts</name>
+ <description>List of Known Hosts for Host Enrichment</description>
+ <value>[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]</value>
+ <display-name>Host Enrichment</display-name>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>enrichment_kafka_start</name>
+ <description>Enrichment Topology Spout Offset</description>
+ <value>UNCOMMITTED_EARLIEST</value>
+ <display-name>Enrichment Offset</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>EARLIEST</value>
+ </entry>
+ <entry>
+ <value>LATEST</value>
+ </entry>
+ <entry>
+ <value>UNCOMMITTED_EARLIEST</value>
+ </entry>
+ <entry>
+ <value>UNCOMMITTED_LATEST</value>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+ <property>
+ <name>enrichment_input_topic</name>
+ <description>Enrichment Input Topic</description>
+ <value>enrichments</value>
+ <display-name>Enrichment Input Topic</display-name>
+ </property>
+ <property>
+ <name>enrichment_output_topic</name>
+ <description>Enrichment Output Topic</description>
+ <value>indexing</value>
+ <display-name>Enrichment Output Topic</display-name>
+ </property>
+ <property>
+ <name>enrichment_error_topic</name>
+ <description>Enrichment Error Topic</description>
+ <value>indexing</value>
+ <display-name>Enrichment Error Topic</display-name>
+ </property>
+ <property>
+ <name>threatintel_error_topic</name>
+ <description>Threat Intel Error Topic</description>
+ <value>indexing</value>
+ <display-name>Threat Intel Error Topic</display-name>
+ </property>
+ <property>
+ <name>enrichment_workers</name>
+ <description>Number of Enrichment Topology Workers</description>
+ <value>1</value>
+ <display-name>Enrichment Workers</display-name>
+ </property>
+ <property>
+ <name>enrichment_acker_executors</name>
+ <description>Number of Enrichment Topology Ackers</description>
+ <value>1</value>
+ <display-name>Enrichment Ackers</display-name>
+ </property>
+ <property>
+ <name>enrichment_topology_worker_childopts</name>
+ <description>Enrichment Topology JVM Options</description>
+ <value/>
+ <display-name>Enrichment childopts</display-name>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>enrichment_topology_max_spout_pending</name>
+ <description>Enrichment Topology Spout Max Pending Tuples</description>
+ <value/>
+ <display-name>Enrichment Max Pending</display-name>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>enrichment_join_cache_size</name>
+ <description>Enrichment Join Bolt Cache Max Size</description>
+ <value>100000</value>
+ <display-name>Enrichment Join Size</display-name>
+ </property>
+ <property>
+ <name>threatintel_join_cache_size</name>
+ <description>Threat Intel Join Bolt Cache Max Size</description>
+ <value>100000</value>
+ <display-name>Threat Intel Join Size</display-name>
+ </property>
+ <property>
+ <name>enrichment_kafka_spout_parallelism</name>
+ <description>Enrichment Topology Kafka Spout Parallelism</description>
+ <value>1</value>
+ <display-name>Enrichment Spout Parallelism</display-name>
+ </property>
+ <property>
+ <name>enrichment_split_parallelism</name>
+ <description>Enrichment Topology Enrichment Split Bolt Parallelism</description>
+ <value>1</value>
+ <display-name>Enrichment Split Parallelism</display-name>
+ </property>
+ <property>
+ <name>enrichment_stellar_parallelism</name>
+ <description>Enrichment Topology Enrichment Stellar Bolt Parallelism</description>
+ <value>1</value>
+ <display-name>Enrichment Stellar Parallelism</display-name>
+ </property>
+ <property>
+ <name>enrichment_join_parallelism</name>
+ <description>Enrichment Topology Enrichment Join Bolt Parallelism</description>
+ <value>1</value>
+ <display-name>Enrichment Join Parallelism</display-name>
+ </property>
+ <property>
+ <name>threat_intel_split_parallelism</name>
+ <description>Enrichment Topology Threat Intel Split Bolt Parallelism</description>
+ <value>1</value>
+ <display-name>Threat Intel Spout Parallelism</display-name>
+ </property>
+ <property>
+ <name>threat_intel_stellar_parallelism</name>
+ <description>Enrichment Topology Threat Intel Stellar Bolt Parallelism</description>
+ <value>1</value>
+ <display-name>Threat Intel Spout Parallelism</display-name>
+ </property>
+ <property>
+ <name>threat_intel_join_parallelism</name>
+ <description>Enrichment Topology Threat Intel Join Bolt Parallelism</description>
+ <value>1</value>
+ <display-name>Threat Intel Join Parallelism</display-name>
+ </property>
+ <property>
+ <name>kafka_writer_parallelism</name>
+ <description>Enrichment Topology Kafka Writer Parallelism</description>
+ <value>1</value>
+ <display-name>Enrichment Kafka Writer Parallelism</display-name>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index a8864e3..87866e8 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -31,12 +31,6 @@
<display-name>Metron apps HDFS dir</display-name>
</property>
<property>
- <name>metron_apps_indexed_hdfs_dir</name>
- <value>{{metron_apps_hdfs_dir}}/indexing/indexed</value>
- <description>Indexing bolts will write to this HDFS directory</description>
- <display-name>Metron apps indexed HDFS dir</display-name>
- </property>
- <property>
<name>metron_zookeeper_config_dir</name>
<value>config/zookeeper</value>
<description>Metron Zookeeper config dir. Relative path to Metron home.</description>
@@ -77,132 +71,17 @@
<display-name>Metron pid dir</display-name>
</property>
<property>
- <name>metron_rest_port</name>
- <value>8082</value>
- <description>Port the REST application runs on</description>
- <display-name>Metron REST port</display-name>
- </property>
- <property>
- <name>metron_management_ui_port</name>
- <value>4200</value>
- <description>Port the Management UI runs on</description>
- <display-name>Management UI port</display-name>
- </property>
- <property>
- <name>metron_jvm_flags</name>
- <description>JVM flags passed to Metron scripts</description>
- <display-name>Metron JVM flags</display-name>
- <value/>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- </property>
- <property>
- <name>metron_spring_profiles_active</name>
- <description>Active Spring profiles</description>
- <display-name>Active Spring profiles</display-name>
- <value/>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- </property>
- <property require-input="true">
- <name>metron_jdbc_driver</name>
- <value></value>
- <description>Class name of the JDBC Driver used by Metron</description>
- <display-name>Metron JDBC Driver</display-name>
- </property>
- <property require-input="true">
- <name>metron_jdbc_url</name>
- <value></value>
- <description>JDBC Connection URL used by Metron</description>
- <display-name>Metron JDBC URL</display-name>
- </property>
- <property require-input="true">
- <name>metron_jdbc_username</name>
- <value></value>
- <description>Metron JDBC Username</description>
- <display-name>Metron JDBC username</display-name>
- </property>
- <property require-input="true">
- <name>metron_jdbc_password</name>
- <value></value>
- <property-type>PASSWORD</property-type>
- <description>Metron JDBC Password</description>
- <display-name>Metron JDBC password</display-name>
- <value-attributes>
- <type>password</type>
- <overridable>false</overridable>
- <hidden>METRON_CLIENT</hidden>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>metron_jdbc_platform</name>
- <value></value>
- <description>Database platform used by Metron. One of: hsqldb, h2, oracle, mysql, postgresql</description>
- <display-name>Metron JDBC platform</display-name>
- </property>
- <property>
- <name>metron_jdbc_client_path</name>
- <description>Path to JDBC jar for selected platform</description>
- <display-name>Metron JDBC client path</display-name>
- <value/>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- </property>
- <property>
- <name>metron_temp_grok_path</name>
- <description>Temporary local file path where grok patterns are written during testing</description>
- <value>./patterns/temp</value>
- <display-name>Metron temp grok path</display-name>
- </property>
- <property>
- <name>metron_default_grok_path</name>
- <description>Default HDFS directory path used when storing Grok patterns</description>
- <value>/apps/metron/patterns</value>
- <display-name>Metron default grok path</display-name>
- </property>
- <property>
- <name>metron_spring_options</name>
- <description>Additional Spring options not included in the rest_application.yml file</description>
- <display-name>Metron Spring options</display-name>
- <value/>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- </property>
- <property>
<name>metron_topic_retention</name>
<description>Kafka Retention in GB</description>
<value>10</value>
<display-name>Topic Retention</display-name>
</property>
<property>
- <name>parsers</name>
- <value>bro,snort,yaf</value>
- <description>Metron parsers to deploy</description>
- <display-name>Metron Parsers</display-name>
- </property>
- <property>
- <name>metron_indexing_topology</name>
- <value>indexing</value>
- <description>The Storm topology name for Indexing</description>
- <display-name>Indexing Topology Name</display-name>
- </property>
- <property>
<name>es_cluster_name</name>
<value>metron</value>
<description>Name of Elasticsearch Cluster</description>
<display-name>Elasticsearch Cluster Name</display-name>
</property>
- <property>
- <name>geoip_url</name>
- <value>http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz</value>
- <description>Location of the GeoIP data to load.</description>
- <display-name>GEOIP Load Datafile URL</display-name>
- </property>
<property require-input="true">
<name>es_hosts</name>
<value></value>
@@ -258,296 +137,6 @@
<value>yyyy.MM.dd.HH</value>
<display-name>Elasticsearch Date Format</display-name>
</property>
- <property>
- <name>parser_error_topic</name>
- <description>Parser Error Topic</description>
- <value>indexing</value>
- <display-name>Parser Error Topic</display-name>
- </property>
- <property>
- <name>enrichment_workers</name>
- <description>Number of Enrichment Topology Workers</description>
- <value>1</value>
- <display-name>Enrichment Workers</display-name>
- </property>
- <property>
- <name>enrichment_acker_executors</name>
- <description>Number of Enrichment Topology Ackers</description>
- <value>1</value>
- <display-name>Enrichment Ackers</display-name>
- </property>
- <property>
- <name>enrichment_topology_worker_childopts</name>
- <description>Enrichment Topology JVM Options</description>
- <value/>
- <display-name>Enrichment childopts</display-name>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- </property>
- <property>
- <name>enrichment_topology_max_spout_pending</name>
- <description>Enrichment Topology Spout Max Pending Tuples</description>
- <value/>
- <display-name>Enrichment Max Pending</display-name>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- </property>
- <property>
- <name>enrichment_kafka_start</name>
- <description>Enrichment Topology Spout Offset</description>
- <value>UNCOMMITTED_EARLIEST</value>
- <display-name>Enrichment Offset</display-name>
- <value-attributes>
- <type>value-list</type>
- <entries>
- <entry>
- <value>EARLIEST</value>
- </entry>
- <entry>
- <value>LATEST</value>
- </entry>
- <entry>
- <value>UNCOMMITTED_EARLIEST</value>
- </entry>
- <entry>
- <value>UNCOMMITTED_LATEST</value>
- </entry>
- </entries>
- <selection-cardinality>1</selection-cardinality>
- </value-attributes>
- </property>
- <property>
- <name>enrichment_input_topic</name>
- <description>Enrichment Input Topic</description>
- <value>enrichments</value>
- <display-name>Enrichment Input Topic</display-name>
- </property>
- <property>
- <name>enrichment_output_topic</name>
- <description>Enrichment Output Topic</description>
- <value>indexing</value>
- <display-name>Enrichment Output Topic</display-name>
- </property>
- <property>
- <name>enrichment_error_topic</name>
- <description>Enrichment Error Topic</description>
- <value>indexing</value>
- <display-name>Enrichment Error Topic</display-name>
- </property>
- <property>
- <name>threatintel_error_topic</name>
- <description>Threat Intel Error Topic</description>
- <value>indexing</value>
- <display-name>Threat Intel Error Topic</display-name>
- </property>
- <property>
- <name>enrichment_join_cache_size</name>
- <description>Enrichment Join Bolt Cache Max Size</description>
- <value>100000</value>
- <display-name>Enrichment Join Size</display-name>
- </property>
- <property>
- <name>threatintel_join_cache_size</name>
- <description>Threat Intel Join Bolt Cache Max Size</description>
- <value>100000</value>
- <display-name>Threat Intel Join Size</display-name>
- </property>
- <property>
- <name>enrichment_host_known_hosts</name>
- <description>List of Known Hosts for Host Enrichment</description>
- <value>[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]</value>
- <display-name>Host Enrichment</display-name>
- <value-attributes>
- <type>content</type>
- </value-attributes>
- </property>
- <property>
- <name>enrichment_kafka_spout_parallelism</name>
- <description>Enrichment Topology Kafka Spout Parallelism</description>
- <value>1</value>
- <display-name>Enrichment Spout Parallelism</display-name>
- </property>
- <property>
- <name>enrichment_split_parallelism</name>
- <description>Enrichment Topology Enrichment Split Bolt Parallelism</description>
- <value>1</value>
- <display-name>Enrichment Split Parallelism</display-name>
- </property>
- <property>
- <name>enrichment_stellar_parallelism</name>
- <description>Enrichment Topology Enrichment Stellar Bolt Parallelism</description>
- <value>1</value>
- <display-name>Enrichment Stellar Parallelism</display-name>
- </property>
- <property>
- <name>enrichment_join_parallelism</name>
- <description>Enrichment Topology Enrichment Join Bolt Parallelism</description>
- <value>1</value>
- <display-name>Enrichment Join Parallelism</display-name>
- </property>
- <property>
- <name>threat_intel_split_parallelism</name>
- <description>Enrichment Topology Threat Intel Split Bolt Parallelism</description>
- <value>1</value>
- <display-name>Threat Intel Spout Parallelism</display-name>
- </property>
- <property>
- <name>threat_intel_stellar_parallelism</name>
- <description>Enrichment Topology Threat Intel Stellar Bolt Parallelism</description>
- <value>1</value>
- <display-name>Threat Intel Spout Parallelism</display-name>
- </property>
- <property>
- <name>threat_intel_join_parallelism</name>
- <description>Enrichment Topology Threat Intel Join Bolt Parallelism</description>
- <value>1</value>
- <display-name>Threat Intel Join Parallelism</display-name>
- </property>
- <property>
- <name>kafka_writer_parallelism</name>
- <description>Enrichment Topology Kafka Writer Parallelism</description>
- <value>1</value>
- <display-name>Enrichment Kafka Writer Parallelism</display-name>
- </property>
- <property>
- <name>indexing_workers</name>
- <description>Number of Indexing Topology Workers</description>
- <value>1</value>
- <display-name>Indexing Workers</display-name>
- </property>
- <property>
- <name>indexing_acker_executors</name>
- <description>Number of Indexing Topology Ackers</description>
- <value>1</value>
- <display-name>Enrichment Ackers</display-name>
- </property>
- <property>
- <name>indexing_topology_worker_childopts</name>
- <description>Indexing Topology JVM Options</description>
- <value/>
- <display-name>Indexing childopts</display-name>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- </property>
- <property>
- <name>indexing_topology_max_spout_pending</name>
- <description>Indexing Topology Spout Max Pending Tuples</description>
- <value/>
- <display-name>Indexing Max Pending</display-name>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- </property>
- <property>
- <name>indexing_kafka_start</name>
- <description>Indexing Topology Spout Offset</description>
- <value>UNCOMMITTED_EARLIEST</value>
- <display-name>Indexing Offset</display-name>
- <value-attributes>
- <type>value-list</type>
- <entries>
- <entry>
- <value>EARLIEST</value>
- </entry>
- <entry>
- <value>LATEST</value>
- </entry>
- <entry>
- <value>UNCOMMITTED_EARLIEST</value>
- </entry>
- <entry>
- <value>UNCOMMITTED_LATEST</value>
- </entry>
- </entries>
- <selection-cardinality>1</selection-cardinality>
- </value-attributes>
- </property>
- <property>
- <name>indexing_input_topic</name>
- <description>Indexing Input Topic</description>
- <value>indexing</value>
- <display-name>Indexing Input Topic</display-name>
- </property>
- <property>
- <name>indexing_error_topic</name>
- <description>Indexing Error Topic</description>
- <value>indexing</value>
- <display-name>Indexing Error Topic</display-name>
- </property>
- <property>
- <name>indexing_writer_class_name</name>
- <description>Indexing Writer Class Name</description>
- <value>org.apache.metron.elasticsearch.writer.ElasticsearchWriter</value>
- <display-name>Indexing Writer Class Name</display-name>
- </property>
- <property>
- <name>bolt_hdfs_rotation_policy</name>
- <description>Class Name of HDFS Rotation Policy</description>
- <value>org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy</value>
- <display-name>HDFS Rotation Policy</display-name>
- </property>
- <property>
- <name>bolt_hdfs_rotation_policy_count</name>
- <description>HDFS Rotation Policy Count</description>
- <value>1</value>
- <display-name>HDFS Rotation Policy Count</display-name>
- </property>
- <property>
- <name>bolt_hdfs_rotation_policy_units</name>
- <description>Frequency of HDFS File Rotation</description>
- <value>DAYS</value>
- <display-name>HDFS Rotation Policy Units</display-name>
- <value-attributes>
- <type>value-list</type>
- <entries>
- <entry>
- <value>SECONDS</value>
- </entry>
- <entry>
- <value>MINUTES</value>
- </entry>
- <entry>
- <value>HOURS</value>
- </entry>
- <entry>
- <value>DAYS</value>
- </entry>
- <entry>
- <value>KB</value>
- </entry>
- <entry>
- <value>MB</value>
- </entry>
- <entry>
- <value>GB</value>
- </entry>
- <entry>
- <value>TB</value>
- </entry>
- </entries>
- <selection-cardinality>1</selection-cardinality>
- </value-attributes>
- </property>
- <property>
- <name>indexing_kafka_spout_parallelism</name>
- <description>Indexing Topology Kafka Spout Parallelism</description>
- <value>1</value>
- <display-name>Indexing Spout Parallelism</display-name>
- </property>
- <property>
- <name>indexing_writer_parallelism</name>
- <description>Indexing Topology Writer Bolt Parallelism</description>
- <value>1</value>
- <display-name>Indexing Writer Parallelism</display-name>
- </property>
- <property>
- <name>hdfs_writer_parallelism</name>
- <description>Indexing Topology HDFS Bolt Parallelism</description>
- <value>1</value>
- <display-name>Indexing HDFS Parallelism</display-name>
- </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
new file mode 100644
index 0000000..e28e328
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
@@ -0,0 +1,165 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>indexing_kafka_start</name>
+ <description>Indexing Topology Spout Offset</description>
+ <value>UNCOMMITTED_EARLIEST</value>
+ <display-name>Indexing Offset</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>EARLIEST</value>
+ </entry>
+ <entry>
+ <value>LATEST</value>
+ </entry>
+ <entry>
+ <value>UNCOMMITTED_EARLIEST</value>
+ </entry>
+ <entry>
+ <value>UNCOMMITTED_LATEST</value>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+ <property>
+ <name>indexing_input_topic</name>
+ <description>Indexing Input Topic</description>
+ <value>indexing</value>
+ <display-name>Indexing Input Topic</display-name>
+ </property>
+ <property>
+ <name>indexing_error_topic</name>
+ <description>Indexing Error Topic</description>
+ <value>indexing</value>
+ <display-name>Indexing Error Topic</display-name>
+ </property>
+ <property>
+ <name>indexing_writer_class_name</name>
+ <description>Indexing Writer Class Name</description>
+ <value>org.apache.metron.elasticsearch.writer.ElasticsearchWriter</value>
+ <display-name>Indexing Writer Class Name</display-name>
+ </property>
+ <property>
+ <name>indexing_workers</name>
+ <description>Number of Indexing Topology Workers</description>
+ <value>1</value>
+ <display-name>Indexing Workers</display-name>
+ </property>
+ <property>
+ <name>indexing_acker_executors</name>
+ <description>Number of Indexing Topology Ackers</description>
+ <value>1</value>
+ <display-name>Enrichment Ackers</display-name>
+ </property>
+ <property>
+ <name>indexing_topology_worker_childopts</name>
+ <description>Indexing Topology JVM Options</description>
+ <value/>
+ <display-name>Indexing childopts</display-name>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>indexing_topology_max_spout_pending</name>
+ <description>Indexing Topology Spout Max Pending Tuples</description>
+ <value/>
+ <display-name>Indexing Max Pending</display-name>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>indexing_kafka_spout_parallelism</name>
+ <description>Indexing Topology Kafka Spout Parallelism</description>
+ <value>1</value>
+ <display-name>Indexing Spout Parallelism</display-name>
+ </property>
+ <property>
+ <name>indexing_writer_parallelism</name>
+ <description>Indexing Topology Writer Bolt Parallelism</description>
+ <value>1</value>
+ <display-name>Indexing Writer Parallelism</display-name>
+ </property>
+ <property>
+ <name>hdfs_writer_parallelism</name>
+ <description>Indexing Topology HDFS Bolt Parallelism</description>
+ <value>1</value>
+ <display-name>Indexing HDFS Parallelism</display-name>
+ </property>
+ <property>
+ <name>metron_apps_indexed_hdfs_dir</name>
+ <value>{{metron_apps_hdfs_dir}}/indexing/indexed</value>
+ <description>Indexing bolts will write to this HDFS directory</description>
+ <display-name>Metron apps indexed HDFS dir</display-name>
+ </property>
+ <property>
+ <name>bolt_hdfs_rotation_policy</name>
+ <description>Class Name of HDFS Rotation Policy</description>
+ <value>org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy</value>
+ <display-name>HDFS Rotation Policy</display-name>
+ </property>
+ <property>
+ <name>bolt_hdfs_rotation_policy_count</name>
+ <description>HDFS Rotation Policy Count</description>
+ <value>1</value>
+ <display-name>HDFS Rotation Policy Count</display-name>
+ </property>
+ <property>
+ <name>bolt_hdfs_rotation_policy_units</name>
+ <description>Frequency of HDFS File Rotation</description>
+ <value>DAYS</value>
+ <display-name>HDFS Rotation Policy Units</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>SECONDS</value>
+ </entry>
+ <entry>
+ <value>MINUTES</value>
+ </entry>
+ <entry>
+ <value>HOURS</value>
+ </entry>
+ <entry>
+ <value>DAYS</value>
+ </entry>
+ <entry>
+ <value>KB</value>
+ </entry>
+ <entry>
+ <value>MB</value>
+ </entry>
+ <entry>
+ <value>GB</value>
+ </entry>
+ <entry>
+ <value>TB</value>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-management-ui-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-management-ui-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-management-ui-env.xml
new file mode 100644
index 0000000..d9adddd
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-management-ui-env.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>metron_management_ui_port</name>
+ <value>4200</value>
+ <description>Port the Management UI runs on</description>
+ <display-name>Management UI port</display-name>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
new file mode 100644
index 0000000..a9a498b
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>parsers</name>
+ <value>bro,snort,yaf</value>
+ <description>Metron parsers to deploy</description>
+ <display-name>Metron Parsers</display-name>
+ </property>
+ <property>
+ <name>parser_error_topic</name>
+ <description>Parser Error Topic</description>
+ <value>indexing</value>
+ <display-name>Parser Error Topic</display-name>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
new file mode 100644
index 0000000..c68f5b2
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -0,0 +1,112 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>metron_rest_port</name>
+ <value>8082</value>
+ <description>Port the REST application runs on</description>
+ <display-name>Metron REST port</display-name>
+ </property>
+ <property>
+ <name>metron_jvm_flags</name>
+ <description>JVM flags passed to Metron scripts</description>
+ <display-name>Metron JVM flags</display-name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>metron_spring_profiles_active</name>
+ <description>Active Spring profiles</description>
+ <display-name>Active Spring profiles</display-name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property require-input="true">
+ <name>metron_jdbc_driver</name>
+ <value></value>
+ <description>Class name of the JDBC Driver used by Metron</description>
+ <display-name>Metron JDBC Driver</display-name>
+ </property>
+ <property require-input="true">
+ <name>metron_jdbc_url</name>
+ <value></value>
+ <description>JDBC Connection URL used by Metron</description>
+ <display-name>Metron JDBC URL</display-name>
+ </property>
+ <property require-input="true">
+ <name>metron_jdbc_username</name>
+ <value></value>
+ <description>Metron JDBC Username</description>
+ <display-name>Metron JDBC username</display-name>
+ </property>
+ <property require-input="true">
+ <name>metron_jdbc_password</name>
+ <value></value>
+ <property-type>PASSWORD</property-type>
+ <description>Metron JDBC Password</description>
+ <display-name>Metron JDBC password</display-name>
+ <value-attributes>
+ <type>password</type>
+ <overridable>false</overridable>
+ <hidden>METRON_CLIENT</hidden>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>metron_jdbc_platform</name>
+ <value></value>
+ <description>Database platform used by Metron. One of: hsqldb, h2, oracle, mysql, postgresql</description>
+ <display-name>Metron JDBC platform</display-name>
+ </property>
+ <property>
+ <name>metron_jdbc_client_path</name>
+ <description>Path to JDBC jar for selected platform</description>
+ <display-name>Metron JDBC client path</display-name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>metron_temp_grok_path</name>
+ <description>Temporary local file path where grok patterns are written during testing</description>
+ <value>./patterns/temp</value>
+ <display-name>Metron temp grok path</display-name>
+ </property>
+ <property>
+ <name>metron_default_grok_path</name>
+ <description>Default HDFS directory path used when storing Grok patterns</description>
+ <value>/apps/metron/patterns</value>
+ <display-name>Metron default grok path</display-name>
+ </property>
+ <property>
+ <name>metron_spring_options</name>
+ <description>Additional Spring options not included in the rest_application.yml file</description>
+ <display-name>Metron Spring options</display-name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index cd86432..b2dda25 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -67,6 +67,9 @@
<script>scripts/parser_master.py</script>
<scriptType>PYTHON</scriptType>
</commandScript>
+ <configuration-dependencies>
+ <config-type>metron-parsers-env</config-type>
+ </configuration-dependencies>
</component>
<component>
@@ -110,6 +113,9 @@
<scriptType>PYTHON</scriptType>
<timeout>600</timeout>
</commandScript>
+ <configuration-dependencies>
+ <config-type>metron-enrichment-env</config-type>
+ </configuration-dependencies>
</component>
<component>
@@ -173,6 +179,9 @@
</commandScript>
</customCommand>
</customCommands>
+ <configuration-dependencies>
+ <config-type>metron-indexing-env</config-type>
+ </configuration-dependencies>
</component>
<component>
@@ -217,6 +226,9 @@
<script>scripts/rest_master.py</script>
<scriptType>PYTHON</scriptType>
</commandScript>
+ <configuration-dependencies>
+ <config-type>metron-rest-env</config-type>
+ </configuration-dependencies>
</component>
<component>
@@ -261,6 +273,10 @@
<script>scripts/management_ui_master.py</script>
<scriptType>PYTHON</scriptType>
</commandScript>
+ <configuration-dependencies>
+ <config-type>metron-rest-env</config-type>
+ <config-type>metron-management-ui-env</config-type>
+ </configuration-dependencies>
</component>
<component>
http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
index 136eade..8ca97ad 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
@@ -39,7 +39,7 @@ class EnrichmentCommands:
raise ValueError("params argument is required for initialization")
self.__params = params
self.__enrichment_topology = params.metron_enrichment_topology
- self.__enrichment_topic = params.metron_enrichment_topic
+ self.__enrichment_topic = params.enrichment_input_topic
self.__kafka_configured = os.path.isfile(self.__params.enrichment_kafka_configured_flag_file)
self.__kafka_acl_configured = os.path.isfile(self.__params.enrichment_kafka_acl_configured_flag_file)
self.__hbase_configured = os.path.isfile(self.__params.enrichment_hbase_configured_flag_file)
@@ -124,14 +124,14 @@ class EnrichmentCommands:
def init_kafka_topics(self):
Logger.info('Creating Kafka topics for enrichment')
# All errors go to indexing topics, so create it here if it's not already
- metron_service.init_kafka_topics(self.__params, [self.__enrichment_topic, self.__params.metron_error_topic])
+ metron_service.init_kafka_topics(self.__params, [self.__enrichment_topic, self.__params.enrichment_error_topic])
self.set_kafka_configured()
def init_kafka_acls(self):
Logger.info('Creating Kafka ACls for enrichment')
# Enrichment topic names matches group
metron_service.init_kafka_acls(self.__params,
- [self.__enrichment_topic, self.__params.metron_error_topic],
+ [self.__enrichment_topic, self.__params.enrichment_error_topic],
[self.__enrichment_topic])
self.set_kafka_acl_configured()
http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index da94c9d..b671c9d 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -28,7 +28,8 @@ import metron_security
# Wrap major operations and functionality in this class
class IndexingCommands:
__params = None
- __indexing = None
+ __indexing_topic = None
+ __indexing_topology = None
__configured = False
__acl_configured = False
__hdfs_perm_configured = False
@@ -37,7 +38,8 @@ class IndexingCommands:
if params is None:
raise ValueError("params argument is required for initialization")
self.__params = params
- self.__indexing = params.metron_indexing_topology
+ self.__indexing_topology = params.metron_indexing_topology
+ self.__indexing_topic = params.indexing_input_topic
self.__configured = os.path.isfile(self.__params.indexing_configured_flag_file)
self.__acl_configured = os.path.isfile(self.__params.indexing_acl_configured_flag_file)
@@ -70,12 +72,12 @@ class IndexingCommands:
def init_kafka_topics(self):
Logger.info('Creating Kafka topics for indexing')
- metron_service.init_kafka_topics(self.__params, [self.__indexing])
+ metron_service.init_kafka_topics(self.__params, [self.__indexing_topic])
def init_kafka_acls(self):
Logger.info('Creating Kafka ACLs')
# Indexed topic names matches the group
- metron_service.init_kafka_acls(self.__params, [self.__indexing], [self.__indexing])
+ metron_service.init_kafka_acls(self.__params, [self.__indexing_topic], [self.__indexing_topic])
def init_hdfs_dir(self):
Logger.info('Setting up HDFS indexing directory')
@@ -94,24 +96,24 @@ class IndexingCommands:
Logger.info('Done creating HDFS indexing directory')
def start_indexing_topology(self):
- Logger.info("Starting Metron indexing topology: {0}".format(self.__indexing))
+ Logger.info("Starting Metron indexing topology: {0}".format(self.__indexing_topology))
start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh \
-s {1} \
-z {2}"""
- Logger.info('Starting ' + self.__indexing)
+ Logger.info('Starting ' + self.__indexing_topology)
if self.__params.security_enabled:
metron_security.kinit(self.__params.kinit_path_local,
self.__params.metron_keytab_path,
self.__params.metron_principal_name,
execute_user=self.__params.metron_user)
- Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing, self.__params.zookeeper_quorum),
+ Execute(start_cmd_template.format(self.__params.metron_home, self.__indexing_topology, self.__params.zookeeper_quorum),
user=self.__params.metron_user)
Logger.info('Finished starting indexing topology')
def stop_indexing_topology(self):
- Logger.info('Stopping ' + self.__indexing)
- stop_cmd = 'storm kill ' + self.__indexing
+ Logger.info('Stopping ' + self.__indexing_topology)
+ stop_cmd = 'storm kill ' + self.__indexing_topology
if self.__params.security_enabled:
metron_security.kinit(self.__params.kinit_path_local,
self.__params.metron_keytab_path,
@@ -146,7 +148,7 @@ class IndexingCommands:
active = True
topologies = metron_service.get_running_topologies(self.__params)
is_running = False
- if self.__indexing in topologies:
- is_running = topologies[self.__indexing] in ['ACTIVE', 'REBALANCING']
+ if self.__indexing_topology in topologies:
+ is_running = topologies[self.__indexing_topology] in ['ACTIVE', 'REBALANCING']
active &= is_running
return active
http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 86ce519..d8d8add 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -39,27 +39,25 @@ tmp_dir = Script.get_tmp_dir()
hostname = config['hostname']
metron_home = status_params.metron_home
parsers = status_params.parsers
-parser_error_topic = config['configurations']['metron-env']['parser_error_topic']
-geoip_url = config['configurations']['metron-env']['geoip_url']
+parser_error_topic = config['configurations']['metron-parsers-env']['parser_error_topic']
geoip_hdfs_dir = "/apps/metron/geo/default/"
-metron_indexing_topology = status_params.metron_indexing_topology
metron_user = status_params.metron_user
metron_group = config['configurations']['metron-env']['metron_group']
metron_log_dir = config['configurations']['metron-env']['metron_log_dir']
metron_pid_dir = config['configurations']['metron-env']['metron_pid_dir']
-metron_rest_port = config['configurations']['metron-env']['metron_rest_port']
-metron_management_ui_port = config['configurations']['metron-env']['metron_management_ui_port']
-metron_jvm_flags = ''
-metron_spring_profiles_active = config['configurations']['metron-env']['metron_spring_profiles_active']
-metron_jdbc_driver = config['configurations']['metron-env']['metron_jdbc_driver']
-metron_jdbc_url = config['configurations']['metron-env']['metron_jdbc_url']
-metron_jdbc_username = config['configurations']['metron-env']['metron_jdbc_username']
-metron_jdbc_password = config['configurations']['metron-env']['metron_jdbc_password']
-metron_jdbc_platform = config['configurations']['metron-env']['metron_jdbc_platform']
-metron_jdbc_client_path = config['configurations']['metron-env']['metron_jdbc_client_path']
-metron_temp_grok_path = config['configurations']['metron-env']['metron_temp_grok_path']
-metron_default_grok_path = config['configurations']['metron-env']['metron_default_grok_path']
-metron_spring_options = config['configurations']['metron-env']['metron_spring_options']
+metron_rest_port = config['configurations']['metron-rest-env']['metron_rest_port']
+metron_management_ui_port = config['configurations']['metron-management-ui-env']['metron_management_ui_port']
+metron_jvm_flags = config['configurations']['metron-rest-env']['metron_jvm_flags']
+metron_spring_profiles_active = config['configurations']['metron-rest-env']['metron_spring_profiles_active']
+metron_jdbc_driver = config['configurations']['metron-rest-env']['metron_jdbc_driver']
+metron_jdbc_url = config['configurations']['metron-rest-env']['metron_jdbc_url']
+metron_jdbc_username = config['configurations']['metron-rest-env']['metron_jdbc_username']
+metron_jdbc_password = config['configurations']['metron-rest-env']['metron_jdbc_password']
+metron_jdbc_platform = config['configurations']['metron-rest-env']['metron_jdbc_platform']
+metron_jdbc_client_path = config['configurations']['metron-rest-env']['metron_jdbc_client_path']
+metron_temp_grok_path = config['configurations']['metron-rest-env']['metron_temp_grok_path']
+metron_default_grok_path = config['configurations']['metron-rest-env']['metron_default_grok_path']
+metron_spring_options = config['configurations']['metron-rest-env']['metron_spring_options']
metron_config_path = metron_home + '/config'
metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir
metron_zookeeper_config_path = status_params.metron_zookeeper_config_path
@@ -131,9 +129,6 @@ if has_kafka_host:
metron_apps_hdfs_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
-# the double "format" is not an error - we are pulling in a jinja-templated param. This is a bit of a hack, but works
-# well enough until we find a better way via Ambari
-metron_apps_indexed_hdfs_dir = format(format(config['configurations']['metron-env']['metron_apps_indexed_hdfs_dir']))
metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention']
local_grok_patterns_dir = format("{metron_home}/patterns")
@@ -175,9 +170,6 @@ threatintel_table = status_params.threatintel_table
threatintel_cf = status_params.threatintel_cf
# Kafka Topics
-metron_enrichment_topology = status_params.metron_enrichment_topology
-metron_enrichment_topic = status_params.metron_enrichment_topic
-metron_error_topic = 'indexing'
ambari_kafka_service_check_topic = 'ambari_kafka_service_check'
consumer_offsets_topic = '__consumer_offsets'
@@ -227,39 +219,48 @@ if security_enabled:
metron_rest_host = default("/clusterHostInfo/metron_rest_hosts", ['localhost'])[0]
# Enrichment
-enrichment_workers = config['configurations']['metron-env']['enrichment_workers']
-enrichment_acker_executors = config['configurations']['metron-env']['enrichment_acker_executors']
-enrichment_topology_worker_childopts = config['configurations']['metron-env']['enrichment_topology_worker_childopts']
-enrichment_topology_max_spout_pending = config['configurations']['metron-env']['enrichment_topology_max_spout_pending']
-enrichment_kafka_start = config['configurations']['metron-env']['enrichment_kafka_start']
-enrichment_input_topic = config['configurations']['metron-env']['enrichment_input_topic']
-enrichment_output_topic = config['configurations']['metron-env']['enrichment_output_topic']
-enrichment_error_topic = config['configurations']['metron-env']['enrichment_error_topic']
-threatintel_error_topic = config['configurations']['metron-env']['threatintel_error_topic']
-enrichment_join_cache_size = config['configurations']['metron-env']['enrichment_join_cache_size']
-threatintel_join_cache_size = config['configurations']['metron-env']['threatintel_join_cache_size']
-enrichment_host_known_hosts = config['configurations']['metron-env']['enrichment_host_known_hosts']
-enrichment_kafka_spout_parallelism = config['configurations']['metron-env']['enrichment_kafka_spout_parallelism']
-enrichment_split_parallelism = config['configurations']['metron-env']['enrichment_split_parallelism']
-enrichment_stellar_parallelism = config['configurations']['metron-env']['enrichment_stellar_parallelism']
-enrichment_join_parallelism = config['configurations']['metron-env']['enrichment_join_parallelism']
-threat_intel_split_parallelism = config['configurations']['metron-env']['threat_intel_split_parallelism']
-threat_intel_stellar_parallelism = config['configurations']['metron-env']['threat_intel_stellar_parallelism']
-threat_intel_join_parallelism = config['configurations']['metron-env']['threat_intel_join_parallelism']
-kafka_writer_parallelism = config['configurations']['metron-env']['kafka_writer_parallelism']
-
-# Threat Intel
-indexing_workers = config['configurations']['metron-env']['indexing_workers']
-indexing_acker_executors = config['configurations']['metron-env']['indexing_acker_executors']
-indexing_topology_worker_childopts = config['configurations']['metron-env']['indexing_topology_worker_childopts']
-indexing_topology_max_spout_pending = config['configurations']['metron-env']['indexing_topology_max_spout_pending']
-indexing_kafka_start = config['configurations']['metron-env']['indexing_kafka_start']
-indexing_input_topic = config['configurations']['metron-env']['indexing_input_topic']
-indexing_error_topic = config['configurations']['metron-env']['indexing_error_topic']
-indexing_writer_class_name = config['configurations']['metron-env']['indexing_writer_class_name']
-bolt_hdfs_rotation_policy = config['configurations']['metron-env']['bolt_hdfs_rotation_policy']
-bolt_hdfs_rotation_policy_units = config['configurations']['metron-env']['bolt_hdfs_rotation_policy_units']
-bolt_hdfs_rotation_policy_count = config['configurations']['metron-env']['bolt_hdfs_rotation_policy_count']
-indexing_kafka_spout_parallelism = config['configurations']['metron-env']['indexing_kafka_spout_parallelism']
-indexing_writer_parallelism = config['configurations']['metron-env']['indexing_writer_parallelism']
-hdfs_writer_parallelism = config['configurations']['metron-env']['hdfs_writer_parallelism']
+geoip_url = config['configurations']['metron-enrichment-env']['geoip_url']
+enrichment_host_known_hosts = config['configurations']['metron-enrichment-env']['enrichment_host_known_hosts']
+enrichment_kafka_start = config['configurations']['metron-enrichment-env']['enrichment_kafka_start']
+enrichment_input_topic = status_params.enrichment_input_topic
+enrichment_output_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic']
+enrichment_error_topic = config['configurations']['metron-enrichment-env']['enrichment_error_topic']
+threatintel_error_topic = config['configurations']['metron-enrichment-env']['threatintel_error_topic']
+metron_enrichment_topology = status_params.metron_enrichment_topology
+enrichment_workers = config['configurations']['metron-enrichment-env']['enrichment_workers']
+enrichment_acker_executors = config['configurations']['metron-enrichment-env']['enrichment_acker_executors']
+enrichment_topology_worker_childopts = config['configurations']['metron-enrichment-env']['enrichment_topology_worker_childopts']
+enrichment_topology_max_spout_pending = config['configurations']['metron-enrichment-env']['enrichment_topology_max_spout_pending']
+enrichment_join_cache_size = config['configurations']['metron-enrichment-env']['enrichment_join_cache_size']
+threatintel_join_cache_size = config['configurations']['metron-enrichment-env']['threatintel_join_cache_size']
+enrichment_kafka_spout_parallelism = config['configurations']['metron-enrichment-env']['enrichment_kafka_spout_parallelism']
+enrichment_split_parallelism = config['configurations']['metron-enrichment-env']['enrichment_split_parallelism']
+enrichment_stellar_parallelism = config['configurations']['metron-enrichment-env']['enrichment_stellar_parallelism']
+enrichment_join_parallelism = config['configurations']['metron-enrichment-env']['enrichment_join_parallelism']
+threat_intel_split_parallelism = config['configurations']['metron-enrichment-env']['threat_intel_split_parallelism']
+threat_intel_stellar_parallelism = config['configurations']['metron-enrichment-env']['threat_intel_stellar_parallelism']
+threat_intel_join_parallelism = config['configurations']['metron-enrichment-env']['threat_intel_join_parallelism']
+kafka_writer_parallelism = config['configurations']['metron-enrichment-env']['kafka_writer_parallelism']
+
+# Indexing
+indexing_kafka_start = config['configurations']['metron-indexing-env']['indexing_kafka_start']
+indexing_input_topic = status_params.indexing_input_topic
+indexing_error_topic = config['configurations']['metron-indexing-env']['indexing_error_topic']
+metron_indexing_topology = status_params.metron_indexing_topology
+indexing_writer_class_name = config['configurations']['metron-indexing-env']['indexing_writer_class_name']
+indexing_workers = config['configurations']['metron-indexing-env']['indexing_workers']
+indexing_acker_executors = config['configurations']['metron-indexing-env']['indexing_acker_executors']
+indexing_topology_worker_childopts = config['configurations']['metron-indexing-env']['indexing_topology_worker_childopts']
+indexing_topology_max_spout_pending = config['configurations']['metron-indexing-env']['indexing_topology_max_spout_pending']
+indexing_kafka_spout_parallelism = config['configurations']['metron-indexing-env']['indexing_kafka_spout_parallelism']
+indexing_writer_parallelism = config['configurations']['metron-indexing-env']['indexing_writer_parallelism']
+hdfs_writer_parallelism = config['configurations']['metron-indexing-env']['hdfs_writer_parallelism']
+
+# the double "format" is not an error - we are pulling in a jinja-templated param. This is a bit of a hack, but works
+# well enough until we find a better way via Ambari
+metron_apps_indexed_hdfs_dir = format(format(config['configurations']['metron-indexing-env']['metron_apps_indexed_hdfs_dir']))
+
+bolt_hdfs_rotation_policy = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy']
+bolt_hdfs_rotation_policy_units = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_units']
+bolt_hdfs_rotation_policy_count = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count']
+
http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 690c301..072a371 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -28,7 +28,7 @@ config = Script.get_config()
metron_user = config['configurations']['metron-env']['metron_user']
# Parsers
-parsers = config['configurations']['metron-env']['parsers']
+parsers = config['configurations']['metron-parsers-env']['parsers']
metron_home = config['configurations']['metron-env']['metron_home']
metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir']
metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}')
@@ -38,7 +38,12 @@ rest_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_rest_
# Enrichment
metron_enrichment_topology = 'enrichment'
-metron_enrichment_topic = 'enrichments'
+enrichment_input_topic = config['configurations']['metron-enrichment-env']['enrichment_input_topic']
+enrichment_kafka_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_kafka_configured'
+enrichment_kafka_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_kafka_acl_configured'
+enrichment_hbase_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_configured'
+enrichment_hbase_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_acl_configured'
+enrichment_geo_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_geo_configured'
enrichment_table = 'enrichment'
enrichment_cf = 't'
@@ -46,18 +51,12 @@ threatintel_table = 'threatintel'
threatintel_cf = 't'
# Indexing
-metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology']
+metron_indexing_topology = 'indexing'
+indexing_input_topic = config['configurations']['metron-indexing-env']['indexing_input_topic']
indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured'
indexing_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_acl_configured'
indexing_hdfs_perm_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_hdfs_perm_configured'
-# Enrichment
-enrichment_kafka_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_kafka_configured'
-enrichment_kafka_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_kafka_acl_configured'
-enrichment_hbase_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_configured'
-enrichment_hbase_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_acl_configured'
-enrichment_geo_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_geo_configured'
-
# Storm
storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr']
http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
index e4cae47..a6e1f1f 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
@@ -89,7 +89,7 @@ class ParserCommands:
# All errors go to indexing topics, so create it here if it's not already
# Getting topics this way is a bit awkward, but I don't want to append to actual list, so copy it
topics = list(self.get_parser_list())
- topics.append(self.__params.metron_error_topic)
+ topics.append(self.__params.enrichment_error_topic)
metron_service.init_kafka_topics(self.__params, topics)
def init_kafka_acls(self):
@@ -97,7 +97,7 @@ class ParserCommands:
# Getting topics this way is a bit awkward, but I don't want to modify the actual list, so copy it
topics = list(self.get_parser_list())
- topics.append(self.__params.metron_error_topic)
+ topics.append(self.__params.enrichment_error_topic)
# Parser group is the parser name + '_parser'
metron_service.init_kafka_acls(self.__params,
topics,
http://git-wip-us.apache.org/repos/asf/metron/blob/a9b01d1e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json
index 31fde1f..dbe2e6a 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json
@@ -35,7 +35,7 @@
"https_property": "metron_rest_port",
"https_default_port": "8082",
"regex": "^(\\d+)$",
- "site": "metron-env"
+ "site": "metron-rest-env"
}
},
{
@@ -50,7 +50,7 @@
"https_property": "metron_management_ui_port",
"https_default_port": "4200",
"regex": "^(\\d+)$",
- "site": "metron-env"
+ "site": "metron-management-ui-env"
}
}
]