You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2018/08/17 15:34:28 UTC
[13/51] [abbrv] metron git commit: METRON-1560 Update MPack to
support Pcap panel (merrimanr) closes apache/metron#1124
METRON-1560 Update MPack to support Pcap panel (merrimanr) closes apache/metron#1124
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e78bc530
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e78bc530
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e78bc530
Branch: refs/heads/master
Commit: e78bc530260aa622e8263cda2b94be99194dffb5
Parents: 8763fba
Author: merrimanr <me...@gmail.com>
Authored: Mon Jul 23 09:08:12 2018 -0500
Committer: rmerriman <me...@gmail.com>
Committed: Mon Jul 23 09:08:12 2018 -0500
----------------------------------------------------------------------
.../CURRENT/configuration/metron-rest-env.xml | 25 ++++++++++++++++++++
.../package/scripts/params/params_linux.py | 5 +++-
.../CURRENT/package/scripts/rest_commands.py | 18 ++++++++++++--
.../METRON/CURRENT/package/templates/metron.j2 | 4 ++++
.../METRON/CURRENT/themes/metron_theme.json | 10 ++++++++
.../src/main/config/rest_application.yml | 6 ++++-
.../src/main/config/pcap.properties | 2 +-
7 files changed, 65 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
index 066b527..20f9767 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -168,4 +168,29 @@
<description>The field name where the threat triage score can be found in the search indices. This setting primarily affects the Alerts UI.</description>
<value>threat:triage:score</value>
</property>
+ <property>
+ <name>pcap_base_path</name>
+ <display-name>Pcap Base Path</display-name>
+ <description>The HDFS path where pcap files are stored. These files are the input to pcap queries.</description>
+ <value>/apps/metron/pcap/input</value>
+ </property>
+ <property>
+ <name>pcap_base_interim_result_path</name>
+ <display-name>Pcap Base Interim Result Path</display-name>
+ <description>The HDFS path where interim results of a pcap query are stored, before they are separated into pages.</description>
+ <value>/apps/metron/pcap/interim</value>
+ </property>
+ <property>
+ <name>pcap_final_output_path</name>
+ <display-name>Pcap Final Output Path</display-name>
+ <description>The HDFS path where the final results of a pcap query are stored.</description>
+ <value>/apps/metron/pcap/output</value>
+ </property>
+ <property>
+ <name>pcap_page_size</name>
+ <display-name>Pcap Page Size</display-name>
+ <description>The number of pcaps written to a page/file as a result of a pcap query.</description>
+ <value>10</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index b8d6123..73d3469 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -386,7 +386,10 @@ bolt_hdfs_rotation_policy_units = config['configurations']['metron-indexing-env'
bolt_hdfs_rotation_policy_count = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count']
# Pcap
-pcap_hdfs_dir = format("{metron_apps_hdfs_dir}/pcap")
+pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path']
+pcap_base_interim_result_path = config['configurations']['metron-rest-env']['pcap_base_interim_result_path']
+pcap_final_output_path = config['configurations']['metron-rest-env']['pcap_final_output_path']
+pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size']
pcap_configured_flag_file = status_params.pcap_configured_flag_file
# MapReduce
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
index 2fd3f0d..463dca1 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@ -117,8 +117,22 @@ class RestCommands:
metron_service.init_kafka_acl_groups(self.__params, groups)
def init_pcap(self):
- Logger.info("Creating HDFS location for Pcap")
- self.__params.HdfsResource(self.__params.pcap_hdfs_dir,
+ Logger.info("Creating HDFS locations for Pcap")
+ self.__params.HdfsResource(self.__params.pcap_base_path,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ group=self.__params.metron_group,
+ mode=0755,
+ )
+ self.__params.HdfsResource(self.__params.pcap_base_interim_result_path,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ group=self.__params.metron_group,
+ mode=0755,
+ )
+ self.__params.HdfsResource(self.__params.pcap_final_output_path,
type="directory",
action="create_on_execute",
owner=self.__params.metron_user,
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
index 049b6ee..278d6f8 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
@@ -40,3 +40,7 @@ SECURITY_ENABLED={{security_enabled|lower}}
KAFKA_SECURITY_PROTOCOL="{{kafka_security_protocol}}"
PARSER_TOPOLOGY_OPTIONS="/home/{{metron_user}}/.storm/storm.config"
METRON_ESCALATION_TOPIC="{{metron_escalation_topic}}"
+PCAP_BASE_PATH="{{pcap_base_path}}"
+PCAP_BASE_INTERIM_RESULT_PATH="{{pcap_base_interim_result_path}}"
+PCAP_FINAL_OUTPUT_PATH="{{pcap_final_output_path}}"
+PCAP_PAGE_SIZE="{{pcap_page_size}}"
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index 17b4460..9f5b04e 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -820,6 +820,10 @@
"subsection-name": "subsection-rest"
},
{
+ "config": "metron-rest-env/pcap_page_size",
+ "subsection-name": "subsection-rest"
+ },
+ {
"config": "metron-management-ui-env/metron_management_ui_port",
"subsection-name": "subsection-management-ui"
},
@@ -1421,6 +1425,12 @@
}
},
{
+ "config": "metron-rest-env/pcap_page_size",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
"config": "metron-management-ui-env/metron_management_ui_port",
"widget": {
"type": "text-field"
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-interface/metron-rest/src/main/config/rest_application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml b/metron-interface/metron-rest/src/main/config/rest_application.yml
index 3999393..7486112 100644
--- a/metron-interface/metron-rest/src/main/config/rest_application.yml
+++ b/metron-interface/metron-rest/src/main/config/rest_application.yml
@@ -55,4 +55,8 @@ kerberos:
keytab: ${METRON_SERVICE_KEYTAB}
pcap:
- pdml.script.path: ${METRON_HOME}/bin/pcap_to_pdml.sh
\ No newline at end of file
+ pdml.script.path: ${METRON_HOME}/bin/pcap_to_pdml.sh
+ base.path: ${PCAP_BASE_PATH}
+ base.interim.result.path: ${PCAP_BASE_INTERIM_RESULT_PATH}
+ final.output.path: ${PCAP_FINAL_OUTPUT_PATH}
+ page.size: ${PCAP_PAGE_SIZE}
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties b/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
index 7160178..848b588 100644
--- a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
+++ b/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
@@ -27,6 +27,6 @@ kafka.pcap.start=UNCOMMITTED_EARLIEST
kafka.pcap.numPackets=1000
kafka.pcap.maxTimeMS=300000
kafka.pcap.ts_scheme=FROM_KEY
-kafka.pcap.out=/apps/metron/pcap
+kafka.pcap.out=/apps/metron/pcap/input
kafka.pcap.ts_granularity=MICROSECONDS
kafka.spout.parallelism=1