You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by an...@apache.org on 2018/10/05 06:40:19 UTC
metron git commit: METRON-1695 Expose pcap properties through Ambari
(anandsubbu) closes apache/metron#1207
Repository: metron
Updated Branches:
refs/heads/master de533063c -> feb9153a6
METRON-1695 Expose pcap properties through Ambari (anandsubbu) closes apache/metron#1207
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/feb9153a
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/feb9153a
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/feb9153a
Branch: refs/heads/master
Commit: feb9153a610e7cc384055c5d07b5c1dcf8b81923
Parents: de53306
Author: anandsubbu <an...@gmail.com>
Authored: Fri Oct 5 12:09:55 2018 +0530
Committer: anandsubbu <an...@apache.org>
Committed: Fri Oct 5 12:09:55 2018 +0530
----------------------------------------------------------------------
.../roles/ambari_config/vars/single_node_vm.yml | 2 +-
metron-deployment/packaging/ambari/.gitignore | 1 +
.../packaging/ambari/metron-mpack/pom.xml | 7 +
.../CURRENT/configuration/metron-pcap-env.xml | 188 +++++++++++++++++++
.../CURRENT/configuration/metron-rest-env.xml | 47 -----
.../common-services/METRON/CURRENT/metainfo.xml | 3 +-
.../package/scripts/params/params_linux.py | 33 +++-
.../package/scripts/params/status_params.py | 2 +-
.../CURRENT/package/scripts/pcap_master.py | 7 +
.../METRON/CURRENT/themes/metron_theme.json | 174 +++++++++++++++--
metron-interface/metron-rest/README.md | 4 +-
metron-platform/metron-pcap-backend/README.md | 3 +
.../src/main/assembly/assembly.xml | 1 +
.../src/main/config/pcap.properties | 9 +-
.../src/main/config/pcap.properties.j2 | 40 ++++
15 files changed, 446 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index 2ec956a..f67522c 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -105,7 +105,7 @@ configurations:
- metron-rest-env:
metron_spring_profiles_active: "dev"
- metron-parsers-env:
- parsers: "bro,snort"
+ parsers: "\"bro,snort,yaf\""
- elastic-site:
index_number_of_shards: 1
index_number_of_replicas: 0
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/.gitignore b/metron-deployment/packaging/ambari/.gitignore
index d2cc43c..242a4da 100644
--- a/metron-deployment/packaging/ambari/.gitignore
+++ b/metron-deployment/packaging/ambari/.gitignore
@@ -6,3 +6,4 @@ hdfs.properties.j2
enrichment.properties.j2
enrichment-splitjoin.properties.j2
enrichment-unified.properties.j2
+pcap.properties.j2
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
index 82386c1..2975963 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -136,6 +136,13 @@
</includes>
<filtering>false</filtering>
</resource>
+ <resource>
+ <directory>${basedir}/../../../../metron-platform/metron-pcap-backend/src/main/config</directory>
+ <includes>
+ <include>pcap.properties.j2</include>
+ </includes>
+ <filtering>false</filtering>
+ </resource>
</resources>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml
new file mode 100644
index 0000000..31f9e8f
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml
@@ -0,0 +1,188 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>pcap_topology_worker_childopts</name>
+ <description>PCAP Topology JVM Options</description>
+ <value/>
+ <display-name>PCAP Topology childopts</display-name>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>pcap_topology_workers</name>
+ <description>Number of PCAP Topology Workers</description>
+ <value>1</value>
+ <display-name>Workers for PCAP Topology</display-name>
+ </property>
+ <property>
+ <name>spout_kafka_topic_pcap</name>
+ <description>PCAP Input Topic</description>
+ <value>pcap</value>
+ <display-name>PCAP Input Topic</display-name>
+ </property>
+ <property>
+ <name>hdfs_sync_every</name>
+ <description>HDFS Sync Every</description>
+ <value>1</value>
+ <display-name>HDFS Sync Every</display-name>
+ </property>
+ <property>
+ <name>hdfs_replication_factor</name>
+ <description>HDFS Replication Factor</description>
+ <value>-1</value>
+ <display-name>HDFS Replication Factor</display-name>
+ </property>
+ <property>
+ <name>kafka_pcap_start</name>
+ <description>PCAP Topology Spout Offset</description>
+ <value>UNCOMMITTED_EARLIEST</value>
+ <display-name>PCAP Topology Offset</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>EARLIEST</value>
+ </entry>
+ <entry>
+ <value>LATEST</value>
+ </entry>
+ <entry>
+ <value>UNCOMMITTED_EARLIEST</value>
+ </entry>
+ <entry>
+ <value>UNCOMMITTED_LATEST</value>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+ <property>
+ <name>kafka_pcap_numpackets</name>
+ <description>Number of Packets</description>
+ <value>1000</value>
+ <display-name>Number of Packets to keep in one file</display-name>
+ </property>
+ <property>
+ <name>kafka_pcap_maxtimems</name>
+ <description>Max Time in milliseconds</description>
+ <value>300000</value>
+ <display-name>Number of packets to keep in terms of duration</display-name>
+ </property>
+ <property>
+ <name>kafka_pcap_tsscheme</name>
+ <description>Kafka PCAP Timestamp Scheme. Use FROM_KEY which works with current tooling</description>
+ <value>FROM_KEY</value>
+ <display-name>Kafka PCAP Timestamp Scheme</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>FROM_KEY</value>
+ </entry>
+ <entry>
+ <value>FROM_VALUE</value>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+ <property>
+ <name>kafka_pcap_out</name>
+ <description>HDFS Directory to store PCAPs</description>
+ <value>/apps/metron/pcap/input</value>
+ <display-name>HDFS Directory to store PCAPs</display-name>
+ </property>
+ <property>
+ <name>kafka_pcap_ts_granularity</name>
+ <description>Granulariy of Timing in Timestamps</description>
+ <value>MICROSECONDS</value>
+ <display-name>Granulariy of Timing in Timestamps</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>MICROSECONDS</value>
+ </entry>
+ <entry>
+ <value>MILLISECONDS</value>
+ </entry>
+ <entry>
+ <value>NANOSECONDS</value>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+ <property>
+ <name>kafka_spout_parallelism</name>
+ <description>PCAP Topology Spout Parallelism</description>
+ <value>1</value>
+ <display-name>PCAP Topology Spout Parallelism</display-name>
+ </property>
+ <property>
+ <name>pcap_base_path</name>
+ <display-name>PCAP Base Path</display-name>
+ <description>The HDFS path where pcap files are stored. These files are the input to pcap queries.</description>
+ <value>/apps/metron/pcap/input</value>
+ </property>
+ <property>
+ <name>pcap_base_interim_result_path</name>
+ <display-name>PCAP Base Interim Result Path</display-name>
+ <description>The HDFS path where interim results of a pcap query are stored, before they are separated into pages.</description>
+ <value>/apps/metron/pcap/interim</value>
+ </property>
+ <property>
+ <name>pcap_final_output_path</name>
+ <display-name>PCAP Final Output Path</display-name>
+ <description>The HDFS path where the final results of a pcap query are stored.</description>
+ <value>/apps/metron/pcap/output</value>
+ </property>
+ <property>
+ <name>pcap_page_size</name>
+ <display-name>PCAP Page Size</display-name>
+ <description>The number of pcaps written to a page/file as a result of a pcap query.</description>
+ <value>10</value>
+ </property>
+ <property>
+ <name>pcap_yarn_queue</name>
+ <display-name>PCAP YARN Queue</display-name>
+ <description>The YARN queue pcap jobs will be submitted to.</description>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>pcap_finalizer_threadpool_size</name>
+ <display-name>PCAP Finalizer Threadpool Size</display-name>
+ <description>The number of threads to use when finalizing Pcap jobs. This affects parallelism
+ around writing out paged files to their final location.
+ If it's a string and ends with "C", then strip the C and treat it as an integral multiple of
+ the number of cores. If it's a string and does not end with a C, then treat it as a number in
+ string form.
+ </description>
+ <value>1</value>
+ <value-attributes>
+ <empty-value-valid>false</empty-value-valid>
+ </value-attributes>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
index 767afa3..f4b2327 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -168,52 +168,5 @@
<description>The field name where the threat triage score can be found in the search indices. This setting primarily affects the Alerts UI.</description>
<value>threat:triage:score</value>
</property>
- <property>
- <name>pcap_base_path</name>
- <display-name>Pcap Base Path</display-name>
- <description>The HDFS path where pcap files are stored. These files are the input to pcap queries.</description>
- <value>/apps/metron/pcap/input</value>
- </property>
- <property>
- <name>pcap_base_interim_result_path</name>
- <display-name>Pcap Base Interim Result Path</display-name>
- <description>The HDFS path where interim results of a pcap query are stored, before they are separated into pages.</description>
- <value>/apps/metron/pcap/interim</value>
- </property>
- <property>
- <name>pcap_final_output_path</name>
- <display-name>Pcap Final Output Path</display-name>
- <description>The HDFS path where the final results of a pcap query are stored.</description>
- <value>/apps/metron/pcap/output</value>
- </property>
- <property>
- <name>pcap_page_size</name>
- <display-name>Pcap Page Size</display-name>
- <description>The number of pcaps written to a page/file as a result of a pcap query.</description>
- <value>10</value>
- </property>
- <property>
- <name>pcap_yarn_queue</name>
- <display-name>Pcap YARN Queue</display-name>
- <description>The YARN queue pcap jobs will be submitted to.</description>
- <value/>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- </property>
- <property>
- <name>pcap_finalizer_threadpool_size</name>
- <display-name>Pcap Finalizer Threadpool Size</display-name>
- <description>The number of threads to use when finalizing Pcap jobs. This affects parallelism
- around writing out paged files to their final location.
- If it's a string and ends with "C", then strip the C and treat it as an integral multiple of
- the number of cores. If it's a string and does not end with a C, then treat it as a number in
- string form.
- </description>
- <value>1</value>
- <value-attributes>
- <empty-value-valid>false</empty-value-valid>
- </value-attributes>
- </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 3c8a4ea..ad1f7a9 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -193,7 +193,7 @@
<timeout>600</timeout>
</commandScript>
<configuration-dependencies>
- <config-type>metron-rest-env</config-type>
+ <config-type>metron-pcap-env</config-type>
</configuration-dependencies>
</component>
@@ -324,6 +324,7 @@
<configuration-dependencies>
<config-type>metron-indexing-env</config-type>
<config-type>metron-rest-env</config-type>
+ <config-type>metron-pcap-env</config-type>
</configuration-dependencies>
</component>
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index dbad44d..458a7be 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -241,6 +241,7 @@ client_jaas_arg = '-Djava.security.auth.login.config=' + metron_home + '/client_
enrichment_topology_worker_childopts = client_jaas_arg if security_enabled else ''
profiler_topology_worker_childopts = client_jaas_arg if security_enabled else ''
indexing_topology_worker_childopts = client_jaas_arg if security_enabled else ''
+pcap_topology_worker_childopts = client_jaas_arg if security_enabled else ''
metron_jvm_flags += (' ' + client_jaas_arg) if security_enabled else ''
topology_auto_credentials = config['configurations']['storm-site'].get('nimbus.credential.renewers.classes', [])
# Needed for storm.config, because it needs Java String
@@ -387,18 +388,32 @@ bolt_hdfs_rotation_policy = config['configurations']['metron-indexing-env']['bol
bolt_hdfs_rotation_policy_units = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_units']
bolt_hdfs_rotation_policy_count = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count']
-# Pcap
-metron_pcap_topology = 'pcap'
-pcap_input_topic = 'pcap'
-pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path']
-pcap_base_interim_result_path = config['configurations']['metron-rest-env']['pcap_base_interim_result_path']
-pcap_final_output_path = config['configurations']['metron-rest-env']['pcap_final_output_path']
-pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size']
-pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue']
-pcap_finalizer_threadpool_size= config['configurations']['metron-rest-env']['pcap_finalizer_threadpool_size']
+# PCAP
+metron_pcap_topology = status_params.metron_pcap_topology
+pcap_input_topic = status_params.pcap_input_topic
+pcap_base_path = config['configurations']['metron-pcap-env']['pcap_base_path']
+pcap_base_interim_result_path = config['configurations']['metron-pcap-env']['pcap_base_interim_result_path']
+pcap_final_output_path = config['configurations']['metron-pcap-env']['pcap_final_output_path']
+pcap_page_size = config['configurations']['metron-pcap-env']['pcap_page_size']
+pcap_yarn_queue = config['configurations']['metron-pcap-env']['pcap_yarn_queue']
+pcap_finalizer_threadpool_size= config['configurations']['metron-pcap-env']['pcap_finalizer_threadpool_size']
pcap_configured_flag_file = status_params.pcap_configured_flag_file
pcap_perm_configured_flag_file = status_params.pcap_perm_configured_flag_file
pcap_acl_configured_flag_file = status_params.pcap_acl_configured_flag_file
+pcap_topology_workers = config['configurations']['metron-pcap-env']['pcap_topology_workers']
+if not len(pcap_topology_worker_childopts) == 0:
+ pcap_topology_worker_childopts += ' '
+pcap_topology_worker_childopts += config['configurations']['metron-pcap-env']['pcap_topology_worker_childopts']
+spout_kafka_topic_pcap = config['configurations']['metron-pcap-env']['spout_kafka_topic_pcap']
+hdfs_sync_every = config['configurations']['metron-pcap-env']['hdfs_sync_every']
+hdfs_replication_factor = config['configurations']['metron-pcap-env']['hdfs_replication_factor']
+kafka_pcap_start = config['configurations']['metron-pcap-env']['kafka_pcap_start']
+kafka_pcap_numpackets = config['configurations']['metron-pcap-env']['kafka_pcap_numpackets']
+kafka_pcap_maxtimems = config['configurations']['metron-pcap-env']['kafka_pcap_maxtimems']
+kafka_pcap_tsscheme = config['configurations']['metron-pcap-env']['kafka_pcap_tsscheme']
+kafka_pcap_out = config['configurations']['metron-pcap-env']['kafka_pcap_out']
+kafka_pcap_ts_granularity = config['configurations']['metron-pcap-env']['kafka_pcap_ts_granularity']
+kafka_spout_parallelism = config['configurations']['metron-pcap-env']['kafka_spout_parallelism']
# MapReduce
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 1ca29b2..aad34a9 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -120,7 +120,7 @@ metron_keytab_path = config['configurations']['metron-env']['metron_service_keyt
# Pcap
metron_pcap_topology = 'pcap'
-pcap_input_topic = 'pcap'
+pcap_input_topic = config['configurations']['metron-pcap-env']['spout_kafka_topic_pcap']
pcap_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_configured'
pcap_perm_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_perm_configured'
pcap_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_acl_configured'
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
index a3bc1b4..109c035 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
@@ -50,6 +50,13 @@ class Pcap(Script):
commands = PcapCommands(params)
+ Logger.info("Running PCAP configure")
+ File(format("{metron_config_path}/pcap.properties"),
+ content=Template("pcap.properties.j2"),
+ owner=params.metron_user,
+ group=params.metron_group
+ )
+
if not commands.is_configured():
commands.init_kafka_topics()
commands.init_hdfs_dir()
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index 2b64f8f..7e6c83a 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -420,6 +420,34 @@
}
]
}
+ },
+ {
+ "name": "metron-pcap",
+ "display-name": "PCAP",
+ "layout": {
+ "tab-columns": "1",
+ "tab-rows": "1",
+ "sections": [
+ {
+ "name": "section-pcap",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-pcap",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
}
]
}
@@ -820,24 +848,72 @@
"subsection-name": "subsection-rest"
},
{
- "config": "metron-rest-env/pcap_page_size",
- "subsection-name": "subsection-rest"
+ "config": "metron-management-ui-env/metron_management_ui_port",
+ "subsection-name": "subsection-management-ui"
},
{
- "config": "metron-rest-env/pcap_yarn_queue",
- "subsection-name": "subsection-rest"
+ "config": "metron-alerts-ui-env/metron_alerts_ui_port",
+ "subsection-name": "subsection-alerts-ui"
},
{
- "config": "metron-rest-env/pcap_finalizer_threadpool_size",
- "subsection-name": "subsection-rest"
+ "config": "metron-pcap-env/pcap_topology_workers",
+ "subsection-name": "subsection-pcap"
},
{
- "config": "metron-management-ui-env/metron_management_ui_port",
- "subsection-name": "subsection-management-ui"
+ "config": "metron-pcap-env/pcap_topology_worker_childopts",
+ "subsection-name": "subsection-pcap"
},
{
- "config": "metron-alerts-ui-env/metron_alerts_ui_port",
- "subsection-name": "subsection-alerts-ui"
+ "config": "metron-pcap-env/spout_kafka_topic_pcap",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/hdfs_sync_every",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/hdfs_replication_factor",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_start",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_numpackets",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_maxtimems",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_tsscheme",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_out",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_ts_granularity",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/kafka_spout_parallelism",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/pcap_page_size",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/pcap_yarn_queue",
+ "subsection-name": "subsection-pcap"
+ },
+ {
+ "config": "metron-pcap-env/pcap_finalizer_threadpool_size",
+ "subsection-name": "subsection-pcap"
}
]
},
@@ -1433,19 +1509,19 @@
}
},
{
- "config": "metron-rest-env/pcap_page_size",
+ "config": "metron-pcap-env/pcap_page_size",
"widget": {
"type": "text-field"
}
},
{
- "config": "metron-rest-env/pcap_yarn_queue",
+ "config": "metron-pcap-env/pcap_yarn_queue",
"widget": {
"type": "text-field"
}
},
{
- "config": "metron-rest-env/pcap_finalizer_threadpool_size",
+ "config": "metron-pcap-env/pcap_finalizer_threadpool_size",
"widget": {
"type": "text-field"
}
@@ -1461,6 +1537,78 @@
"widget": {
"type": "text-field"
}
+ },
+ {
+ "config": "metron-pcap-env/pcap_topology_workers",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/pcap_topology_worker_childopts",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/spout_kafka_topic_pcap",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/hdfs_sync_every",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/hdfs_replication_factor",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_start",
+ "widget": {
+ "type": "combo"
+ }
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_numpackets",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_maxtimems",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_tsscheme",
+ "widget": {
+ "type": "combo"
+ }
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_out",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "metron-pcap-env/kafka_pcap_ts_granularity",
+ "widget": {
+ "type": "combo"
+ }
+ },
+ {
+ "config": "metron-pcap-env/kafka_spout_parallelism",
+ "widget": {
+ "type": "text-field"
+ }
}
]
}
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index 2ce9522..7f00cde 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -221,7 +221,7 @@ The REST application uses a Java Process object to call out to the `pcap_to_pdml
Out of the box it is a simple wrapper around the tshark command to transform raw pcap data to PDML. However it can be extended to do additional processing as long as the expected input/output is maintained.
REST will supply the script with raw pcap data through standard in and expects PDML data serialized as XML.
-Pcap query jobs can be configured for submission to a YARN queue. This setting is exposed as the Spring property `pcap.yarn.queue`. If configured, the REST application will set the `mapreduce.job.queuename` Hadoop property to that value.
+Pcap query jobs can be configured for submission to a YARN queue. This setting is exposed as the Spring property `pcap.yarn.queue` and can be set in the PCAP tab under Metron service -> Configs in Ambari. If configured, the REST application will set the `mapreduce.job.queuename` Hadoop property to that value.
It is highly recommended that a dedicated YARN queue be created and configured for Pcap queries to prevent a job from consuming too many cluster resources. More information about setting up YARN queues can be found [here](https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html#Setting_up_queues).
Pcap query results are stored in HDFS. The location of query results when run through the REST app is determined by a couple factors. The root of Pcap query results defaults to `/apps/metron/pcap/output` but can be changed with the
@@ -234,7 +234,7 @@ periodically delete files and directories under the Pcap query results root.
Users should also be mindful of date ranges used in queries so they don't produce result sets that are too large. Currently there are no limits enforced on date ranges.
-Queries can also be configured on a global level for setting the number of results per page via a Spring property `pcap.page.size`. By default, this value is set to 10 pcaps per page, but you may choose to set this value higher
+Queries can also be configured on a global level for setting the number of results per page via a Spring property `pcap.page.size`. This property can be set in the PCAP tab under Metron service -> Configs, in Ambari. By default, this value is set to 10 pcaps per page, but you may choose to set this value higher
based on observing frequenetly-run query result sizes. This setting works in conjunction with the property for setting finalizer threadpool size when optimizing query performance.
Pcap query jobs have a finalization routine that writes their results out to HDFS in pages. Depending on the size of your pcaps, the number or results typically returned, page sizing (described above), and available CPU cores for running
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-platform/metron-pcap-backend/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/README.md b/metron-platform/metron-pcap-backend/README.md
index 031328d..49063e5 100644
--- a/metron-platform/metron-pcap-backend/README.md
+++ b/metron-platform/metron-pcap-backend/README.md
@@ -70,6 +70,9 @@ sequence files.
## Configuration
+The configuration properties for PCAP sensor is managed via Ambari at Services -> Metron -> Config -> PCAP tab.
+Note that changes to PCAP sensor config properties via Ambari requires restarting the Metron PCAP service.
+
The configuration file for the Flux topology is located at
`$METRON_HOME/config/pcap.properties` and the possible options
are as follows:
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml b/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml
index 5b21e10..f38582d 100644
--- a/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-pcap-backend/src/main/assembly/assembly.xml
@@ -25,6 +25,7 @@
<excludes>
<exclude>**/*.formatted</exclude>
<exclude>**/*.filtered</exclude>
+ <exclude>**/*.j2</exclude>
</excludes>
<fileMode>0644</fileMode>
<lineEnding>unix</lineEnding>
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties b/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
index 848b588..7ee8a95 100644
--- a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
+++ b/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
@@ -14,19 +14,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-spout.kafka.topic.pcap=pcap
+##### Storm #####
topology.worker.childopts=
topology.auto-credentials=[]
topology.workers=1
+
+##### Kafka #####
+spout.kafka.topic.pcap=pcap
kafka.zk=node1:2181
hdfs.sync.every=1
hdfs.replication.factor=-1
kafka.security.protocol=PLAINTEXT
+
# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
kafka.pcap.start=UNCOMMITTED_EARLIEST
+
kafka.pcap.numPackets=1000
kafka.pcap.maxTimeMS=300000
kafka.pcap.ts_scheme=FROM_KEY
kafka.pcap.out=/apps/metron/pcap/input
+
+##### Parallelism #####
kafka.pcap.ts_granularity=MICROSECONDS
kafka.spout.parallelism=1
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties.j2
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties.j2 b/metron-platform/metron-pcap-backend/src/main/config/pcap.properties.j2
new file mode 100644
index 0000000..21f676a
--- /dev/null
+++ b/metron-platform/metron-pcap-backend/src/main/config/pcap.properties.j2
@@ -0,0 +1,40 @@
+{# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+##### Storm #####
+topology.worker.childopts={{pcap_topology_worker_childopts}}
+topology.auto-credentials={{topology_auto_credentials}}
+topology.workers={{pcap_topology_workers}}
+
+##### Kafka #####
+spout.kafka.topic.pcap={{spout_kafka_topic_pcap}}
+kafka.zk={{zookeeper_quorum}}
+hdfs.sync.every={{hdfs_sync_every}}
+hdfs.replication.factor={{hdfs_replication_factor}}
+kafka.security.protocol={{kafka_security_protocol}}
+
+# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
+kafka.pcap.start={{kafka_pcap_start}}
+
+kafka.pcap.numPackets={{kafka_pcap_numpackets}}
+kafka.pcap.maxTimeMS={{kafka_pcap_maxtimems}}
+kafka.pcap.ts_scheme={{kafka_pcap_tsscheme}}
+kafka.pcap.out={{kafka_pcap_out}}
+
+##### Parallelism #####
+kafka.pcap.ts_granularity={{kafka_pcap_ts_granularity}}
+kafka.spout.parallelism={{kafka_spout_parallelism}}
\ No newline at end of file