You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ma...@apache.org on 2017/06/01 21:41:26 UTC
[07/44] metron git commit: METRON-795 Install Metron REST with Ambari
MPack (merrimanr) closes apache/incubator-metron#500
METRON-795 Install Metron REST with Ambari MPack (merrimanr) closes apache/incubator-metron#500
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/716bda32
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/716bda32
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/716bda32
Branch: refs/heads/Metron_0.4.0
Commit: 716bda326f26b60c98318cdc1cc949b351a0b71a
Parents: 38d26d4
Author: merrimanr <me...@gmail.com>
Authored: Thu May 4 15:57:14 2017 -0500
Committer: merrimanr <me...@gmail.com>
Committed: Thu May 4 15:57:14 2017 -0500
----------------------------------------------------------------------
.../METRON/CURRENT/role_command_order.json | 2 +
.../METRON/CURRENT/configuration/metron-env.xml | 103 ++++++++
.../common-services/METRON/CURRENT/metainfo.xml | 47 ++++
.../package/scripts/params/params_linux.py | 17 ++
.../package/scripts/params/status_params.py | 1 +
.../CURRENT/package/scripts/rest_commands.py | 69 ++++++
.../CURRENT/package/scripts/rest_master.py | 83 +++++++
.../METRON/CURRENT/package/templates/metron.j2 | 39 +++
.../METRON/CURRENT/quicklinks/quicklinks.json | 15 ++
.../METRON/CURRENT/service_advisor.py | 5 +
.../docker/rpm-docker/SPECS/metron.spec | 35 +++
.../packaging/docker/rpm-docker/pom.xml | 6 +
.../roles/ambari_config/vars/single_node_vm.yml | 9 +-
.../roles/ambari_config/vars/small_cluster.yml | 3 +-
metron-interface/metron-rest/README.md | 237 ++++++++++++-------
.../src/main/config/rest_application.yml | 54 +++++
.../metron/rest/controller/HdfsController.java | 4 +-
.../src/main/resources/application-vagrant.yml | 2 +-
.../src/main/resources/schema-mysql.sql | 3 -
.../metron-rest/src/main/scripts/metron-rest | 160 +++++++++++++
.../src/main/scripts/start_metron_rest.sh | 25 --
.../metron-rest/src/test/resources/README.vm | 223 +++++++++++------
22 files changed, 953 insertions(+), 189 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
index 9c71887..829758f 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
@@ -5,10 +5,12 @@
"_comment" : "dependencies for all cases",
"METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"],
"METRON_ENRICHMENT-INSTALL": ["METRON_INDEXING-INSTALL"],
+ "METRON_REST-INSTALL": ["METRON_PARSERS-INSTALL"],
"METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_ENRICHMENT_MASTER-START"],
"METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
"METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"],
"METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"],
+ "METRON_REST-START": ["KAFKA_BROKER-START","STORM_REST_API-START","ZOOKEEPER_SERVER-START","NAMENODE-START","METRON_PARSERS-INSTALL","METRON_INDEXING-INSTALL","METRON_ENRICHMENT-INSTALL"],
"METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"]
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index d09614b..ebc2d73 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -65,6 +65,109 @@
</value-attributes>
</property>
<property>
+ <name>metron_log_dir</name>
+ <value>/var/log/metron</value>
+ <description>Log directory for metron</description>
+ <display-name>Metron log dir</display-name>
+ </property>
+ <property>
+ <name>metron_pid_dir</name>
+ <value>/var/run/metron</value>
+ <description>The directory for metron pid files</description>
+ <display-name>Metron pid dir</display-name>
+ </property>
+ <property>
+ <name>metron_rest_port</name>
+ <value>8082</value>
+ <description>Port the REST application runs on</description>
+ <display-name>Metron REST port</display-name>
+ </property>
+ <property>
+ <name>metron_jvm_flags</name>
+ <description>JVM flags passed to Metron scripts</description>
+ <display-name>Metron JVM flags</display-name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>metron_spring_profiles_active</name>
+ <description>Active Spring profiles</description>
+ <display-name>Active Spring profiles</display-name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property require-input="true">
+ <name>metron_jdbc_driver</name>
+ <value></value>
+ <description>Class name of the JDBC Driver used by Metron</description>
+ <display-name>Metron JDBC Driver</display-name>
+ </property>
+ <property require-input="true">
+ <name>metron_jdbc_url</name>
+ <value></value>
+ <description>JDBC Connection URL used by Metron</description>
+ <display-name>Metron JDBC URL</display-name>
+ </property>
+ <property require-input="true">
+ <name>metron_jdbc_username</name>
+ <value></value>
+ <description>Metron JDBC Username</description>
+ <display-name>Metron JDBC username</display-name>
+ </property>
+ <property require-input="true">
+ <name>metron_jdbc_password</name>
+ <value></value>
+ <property-type>PASSWORD</property-type>
+ <description>Metron JDBC Password</description>
+ <display-name>Metron JDBC password</display-name>
+ <value-attributes>
+ <type>password</type>
+ <overridable>false</overridable>
+ <hidden>METRON_CLIENT</hidden>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property require-input="true">
+ <name>metron_jdbc_platform</name>
+ <value></value>
+ <description>Database platform used by Metron. One of: hsqldb, h2, oracle, mysql, postgresql</description>
+ <display-name>Metron JDBC platform</display-name>
+ </property>
+ <property>
+ <name>metron_jdbc_client_path</name>
+ <description>Path to JDBC jar for selected platform</description>
+ <display-name>Metron JDBC client path</display-name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>metron_temp_grok_path</name>
+ <description>Temporary local file path where grok patterns are written during testing</description>
+ <value>./patterns/temp</value>
+ <display-name>Metron temp grok path</display-name>
+ </property>
+ <property>
+ <name>metron_default_grok_path</name>
+ <description>Default HDFS directory path used when storing Grok patterns</description>
+ <value>/apps/metron/patterns</value>
+ <display-name>Metron default grok path</display-name>
+ </property>
+ <property>
+ <name>metron_spring_options</name>
+ <description>Additional Spring options not included in the rest_application.yml file</description>
+ <display-name>Metron Spring options</display-name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
<name>metron_topic_retention</name>
<description>Kafka Retention in GB</description>
<value>10</value>
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index a4e82e2..e0fac4c 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -176,6 +176,50 @@
</component>
<component>
+ <name>METRON_REST</name>
+ <displayName>Metron REST</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <reassignAllowed>false</reassignAllowed>
+ <clientsToUpdateConfigs></clientsToUpdateConfigs>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>STORM/SUPERVISOR</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>KAFKA/KAFKA_BROKER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/rest_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
<name>METRON_CLIENT</name>
<displayName>Metron Client</displayName>
<category>CLIENT</category>
@@ -222,6 +266,9 @@
<package>
<name>metron-pcap</name>
</package>
+ <package>
+ <name>metron-rest</name>
+ </package>
</packages>
</osSpecific>
<osSpecific>
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index f2ab12b..7f34703 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -44,11 +44,26 @@ geoip_hdfs_dir = "/apps/metron/geo/default/"
metron_indexing_topology = status_params.metron_indexing_topology
metron_user = status_params.metron_user
metron_group = config['configurations']['metron-env']['metron_group']
+metron_log_dir = config['configurations']['metron-env']['metron_log_dir']
+metron_pid_dir = config['configurations']['metron-env']['metron_pid_dir']
+metron_rest_port = config['configurations']['metron-env']['metron_rest_port']
+metron_jvm_flags = ''
+metron_spring_profiles_active = config['configurations']['metron-env']['metron_spring_profiles_active']
+metron_jdbc_driver = config['configurations']['metron-env']['metron_jdbc_driver']
+metron_jdbc_url = config['configurations']['metron-env']['metron_jdbc_url']
+metron_jdbc_username = config['configurations']['metron-env']['metron_jdbc_username']
+metron_jdbc_password = config['configurations']['metron-env']['metron_jdbc_password']
+metron_jdbc_platform = config['configurations']['metron-env']['metron_jdbc_platform']
+metron_jdbc_client_path = config['configurations']['metron-env']['metron_jdbc_client_path']
+metron_temp_grok_path = config['configurations']['metron-env']['metron_temp_grok_path']
+metron_default_grok_path = config['configurations']['metron-env']['metron_default_grok_path']
+metron_spring_options = config['configurations']['metron-env']['metron_spring_options']
metron_config_path = metron_home + '/config'
metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir
metron_zookeeper_config_path = status_params.metron_zookeeper_config_path
parsers_configured_flag_file = status_params.parsers_configured_flag_file
parsers_acl_configured_flag_file = status_params.parsers_acl_configured_flag_file
+rest_acl_configured_flag_file = status_params.rest_acl_configured_flag_file
enrichment_kafka_configured_flag_file = status_params.enrichment_kafka_configured_flag_file
enrichment_kafka_acl_configured_flag_file = status_params.enrichment_kafka_acl_configured_flag_file
enrichment_hbase_configured_flag_file = status_params.enrichment_hbase_configured_flag_file
@@ -160,6 +175,8 @@ threatintel_cf = status_params.threatintel_cf
metron_enrichment_topology = status_params.metron_enrichment_topology
metron_enrichment_topic = status_params.metron_enrichment_topic
metron_error_topic = 'indexing'
+ambari_kafka_service_check_topic = 'ambari_kafka_service_check'
+consumer_offsets_topic = '__consumer_offsets'
# ES Templates
bro_index_path = tmp_dir + "/bro_index.template"
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 154b3de..690c301 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -34,6 +34,7 @@ metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zoo
metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}')
parsers_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_configured'
parsers_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_parsers_acl_configured'
+rest_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_rest_acl_configured'
# Enrichment
metron_enrichment_topology = 'enrichment'
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
new file mode 100755
index 0000000..fe5fa6e
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+from resource_management.libraries.functions.format import format
+
+import metron_service
+
+# Wrap major operations and functionality in this class
+class RestCommands:
+ __params = None
+ __acl_configured = False
+
+ def __init__(self, params):
+ if params is None:
+ raise ValueError("params argument is required for initialization")
+ self.__params = params
+ self.__acl_configured = os.path.isfile(self.__params.rest_acl_configured_flag_file)
+
+ def is_acl_configured(self):
+ return self.__acl_configured
+
+ def set_acl_configured(self):
+ File(self.__params.rest_acl_configured_flag_file,
+ content="",
+ owner=self.__params.metron_user,
+ mode=0755)
+
+ def init_kafka_acls(self):
+ Logger.info('Creating Kafka ACLs for rest')
+ # The following topics must be permissioned for the rest application list operation
+ topics = [self.__params.ambari_kafka_service_check_topic, self.__params.consumer_offsets_topic]
+ metron_service.init_kafka_acls(self.__params, topics, ['metron-rest'])
+
+ def start_rest_application(self):
+ Logger.info('Starting REST application')
+ command = format("service metron-rest start {metron_jdbc_password!p}")
+ Execute(command)
+ Logger.info('Done starting REST application')
+
+ def stop_rest_application(self):
+ Logger.info('Stopping REST application')
+ Execute("service metron-rest stop")
+ Logger.info('Done stopping REST application')
+
+ def restart_rest_application(self, env):
+ Logger.info('Restarting the REST application')
+ command = format("service metron-rest restart {metron_jdbc_password!p}")
+ Execute(command)
+ Logger.info('Done restarting the REST application')
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
new file mode 100755
index 0000000..a097dff
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
@@ -0,0 +1,83 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.exceptions import ExecutionFailed
+from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import File
+from resource_management.core.source import Template
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.script import Script
+from resource_management.core.resources.system import Execute
+
+from rest_commands import RestCommands
+
+
+class RestMaster(Script):
+
+ def install(self, env):
+ from params import params
+ env.set_params(params)
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ from params import params
+ if params.security_enabled:
+ params.metron_jvm_flags = format('-Djava.security.auth.login.config={client_jaas_path}')
+
+ env.set_params(params)
+ File(format("/etc/sysconfig/metron"),
+ content=Template("metron.j2")
+ )
+
+ commands = RestCommands(params)
+ if params.security_enabled and not commands.is_acl_configured():
+ commands.init_kafka_acls()
+ commands.set_acl_configured()
+
+ def start(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = RestCommands(params)
+ commands.start_rest_application()
+
+ def stop(self, env, upgrade_type=None):
+ from params import params
+ env.set_params(params)
+ commands = RestCommands(params)
+ commands.stop_rest_application()
+
+ def status(self, env):
+ status_cmd = format('service metron-rest status')
+ try:
+ Execute(status_cmd)
+ except ExecutionFailed:
+ raise ComponentIsNotRunning()
+
+ def restart(self, env):
+ from params import params
+ env.set_params(params)
+ self.configure(env)
+ commands = RestCommands(params)
+ commands.restart_rest_application(env)
+
+
+if __name__ == "__main__":
+ RestMaster().execute()
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
new file mode 100644
index 0000000..7543a67
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+METRON_HOME="{{metron_home}}"
+METRON_USER="{{metron_user}}"
+METRON_LOG_DIR="{{metron_log_dir}}"
+METRON_PID_DIR="{{metron_pid_dir}}"
+METRON_REST_PORT={{metron_rest_port}}
+METRON_JVMFLAGS="{{metron_jvm_flags}}"
+METRON_SPRING_PROFILES_ACTIVE="{{metron_spring_profiles_active}}"
+METRON_JDBC_DRIVER="{{metron_jdbc_driver}}"
+METRON_JDBC_URL="{{metron_jdbc_url}}"
+METRON_JDBC_USERNAME="{{metron_jdbc_username}}"
+METRON_JDBC_PLATFORM="{{metron_jdbc_platform}}"
+METRON_JDBC_CLIENT_PATH="{{metron_jdbc_client_path}}"
+METRON_TEMP_GROK_PATH="{{metron_temp_grok_path}}"
+METRON_DEFAULT_GROK_PATH="{{metron_default_grok_path}}"
+METRON_SPRING_OPTIONS="{{metron_spring_options}}"
+ZOOKEEPER="{{zookeeper_quorum}}"
+BROKERLIST="{{kafka_brokers}}"
+HDFS_URL="{{default_fs}}"
+STORM_REST_URL="{{storm_rest_addr}}"
+SECURITY_ENABLED={{security_enabled|lower}}
+{% if metron_principal_name is defined %}METRON_PRINCIPAL_NAME="{{metron_principal_name}}"
+{% endif %}
+{% if metron_keytab_path is defined %}METRON_SERVICE_KEYTAB="{{metron_keytab_path}}"
+{% endif %}
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json
index ee1b225..f9b7091 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/quicklinks/quicklinks.json
@@ -22,6 +22,21 @@
"regex": "^(\\d+)$",
"site": "storm-site"
}
+ },
+ {
+ "name": "metron_rest_ui",
+ "label": "Swagger UI",
+ "requires_user_name": "false",
+ "component_name": "METRON_REST",
+ "url":"%@://%@:%@/swagger-ui.html",
+ "port":{
+ "http_property": "metron_rest_port",
+ "http_default_port": "8082",
+ "https_property": "metron_rest_port",
+ "https_default_port": "8082",
+ "regex": "^(\\d+)$",
+ "site": "metron-env"
+ }
}
]
}
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
index 23aebb4..464857a 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
@@ -44,6 +44,7 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
metronParsersHost = self.getHosts(componentsList, "METRON_PARSERS")[0]
metronEnrichmentMaster = self.getHosts(componentsList, "METRON_ENRICHMENT_MASTER")[0]
metronIndexingHost = self.getHosts(componentsList, "METRON_INDEXING")[0]
+ metronRESTHost = self.getHosts(componentsList, "METRON_REST")[0]
hbaseClientHosts = self.getHosts(componentsList, "HBASE_CLIENT")
hdfsClientHosts = self.getHosts(componentsList, "HDFS_CLIENT")
@@ -63,6 +64,10 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
message = "Metron must be colocated with an instance of STORM SUPERVISOR"
items.append({ "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'METRON_PARSERS', "host": metronParsersHost })
+ if metronRESTHost not in stormSupervisors:
+ message = "Metron REST must be colocated with an instance of STORM SUPERVISOR"
+ items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_REST', "host": metronRESTHost })
+
if metronParsersHost != metronEnrichmentMaster:
message = "Metron Enrichment Master must be co-located with Metron Parsers on {0}".format(metronParsersHost)
items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_ENRICHMENT_MASTER', "host": metronEnrichmentMaster })
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index eb30cac..76129c3 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -50,6 +50,7 @@ Source5: metron-enrichment-%{full_version}-archive.tar.gz
Source6: metron-indexing-%{full_version}-archive.tar.gz
Source7: metron-pcap-backend-%{full_version}-archive.tar.gz
Source8: metron-profiler-%{full_version}-archive.tar.gz
+Source9: metron-rest-%{full_version}-archive.tar.gz
Source10: metron-config-%{full_version}-archive.tar.gz
%description
@@ -70,6 +71,7 @@ rm -rf %{_builddir}/*
%install
rm -rf %{buildroot}
mkdir -p %{buildroot}%{metron_home}
+mkdir -p %{buildroot}/etc/init.d
# copy source files and untar
tar -xzf %{SOURCE0} -C %{buildroot}%{metron_home}
@@ -81,8 +83,11 @@ tar -xzf %{SOURCE5} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE6} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE7} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE8} -C %{buildroot}%{metron_home}
+tar -xzf %{SOURCE9} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE10} -C %{buildroot}%{metron_home}
+install %{buildroot}%{metron_home}/bin/metron-rest %{buildroot}/etc/init.d/
+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%package common
@@ -320,6 +325,34 @@ This package installs the Metron Profiler %{metron_home}
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+%package rest
+Summary: Metron Rest
+Group: Applications/Internet
+Provides: rest = %{version}
+
+%description rest
+This package installs the Metron Rest %{metron_home}
+
+%files rest
+%defattr(-,root,root,755)
+%dir %{metron_root}
+%dir %{metron_home}
+%dir %{metron_home}/config
+%dir %{metron_home}/bin
+%dir %{metron_home}/lib
+%{metron_home}/config/rest_application.yml
+%{metron_home}/bin/metron-rest
+/etc/init.d/metron-rest
+%attr(0644,root,root) %{metron_home}/lib/metron-rest-%{full_version}.jar
+
+%post rest
+chkconfig --add metron-rest
+
+%preun rest
+chkconfig --del metron-rest
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
%package config
Summary: Metron Management UI
Group: Applications/Internet
@@ -366,6 +399,8 @@ This package installs the Metron Management UI %{metron_home}
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%changelog
+* Thu May 4 2017 Ryan Merriman <me...@gmail.com> - 0.4.0
+- Added REST
* Fri Apr 28 2017 Apache Metron <de...@metron.apache.org> - 0.4.0
- Add Zeppelin Connection Report Dashboard
* Thu Jan 19 2017 Justin Leet <ju...@gmail.com> - 0.3.1
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-deployment/packaging/docker/rpm-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index ec22223..8feeb2b 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -156,6 +156,12 @@
</includes>
</resource>
<resource>
+ <directory>${metron_dir}/metron-interface/metron-rest/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
<directory>${metron_dir}/metron-interface/metron-config/target/</directory>
<includes>
<include>*.tar.gz</include>
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
index 46a486e..fe25eed 100644
--- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
@@ -33,11 +33,12 @@ kibana_master: [KIBANA_MASTER]
metron_indexing: [METRON_INDEXING]
metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
metron_parsers : [METRON_PARSERS]
+metron_rest: [METRON_REST]
metron_components: >
{{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) |
union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) | union(metron_indexing) |
- union(metron_enrichment_master) | union(metron_parsers) | union(es_master) }}
+ union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(es_master) }}
cluster_name: "metron_cluster"
blueprint_name: "metron_blueprint"
@@ -90,6 +91,7 @@ configurations:
delete.topic.enable: "true"
- metron-env:
parsers: "bro,snort"
+ metron_spring_profiles_active: "dev"
- elastic-site:
index_number_of_shards: 1
index_number_of_replicas: 0
@@ -103,6 +105,11 @@ required_configurations:
storm_rest_addr: "{{ groups.ambari_slave[0] }}:8744"
es_hosts: "{{ groups.search | join(',') }}"
zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995"
+ metron_jdbc_driver: "org.h2.Driver"
+ metron_jdbc_url: "jdbc:h2:file:~/metrondb"
+ metron_jdbc_username: "root"
+ metron_jdbc_password: "root"
+ metron_jdbc_platform: "h2"
- kibana-env:
kibana_pid_dir: /var/run/kibana
kibana_es_url: http://{{ groups.search[0] }}:9200
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-deployment/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
index 2f15740..76d7761 100644
--- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml
+++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
@@ -33,6 +33,7 @@ kibana_master: [KIBANA_MASTER]
metron_indexing: [METRON_INDEXING]
metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
metron_parsers : [METRON_PARSERS]
+metron_rest: [METRON_REST]
master_1_components: "{{ hadoop_master | union(hadoop_clients) | union(es_slave) }}"
master_1_host:
@@ -41,7 +42,7 @@ master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_ma
master_2_host:
- "{{groups.ambari_slave[1]}}"
metron_components: >
- {{ metron_indexing | union(metron_enrichment_master) | union(metron_parsers) | union(hadoop_slave) | union(storm_slave) |
+ {{ metron_indexing | union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(hadoop_slave) | union(storm_slave) |
union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}
metron_host:
- "{{ groups.metron[0] }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index 89984ed..93beb0c 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -7,76 +7,165 @@ This module provides a RESTful API for interacting with Metron.
* A running Metron cluster
* Java 8 installed
* Storm CLI and Metron topology scripts (start_parser_topology.sh, start_enrichment_topology.sh, start_elasticsearch_topology.sh) installed
+* A relational database
## Installation
+
+### From Source
+
1. Package the application with Maven:
- ```
- mvn clean package
- ```
+```
+mvn clean package
+```
+
+1. Untar the archive in the $METRON_HOME directory. The directory structure will look like:
+```
+config
+ rest_application.yml
+bin
+ metron-rest
+lib
+ metron-rest-$METRON_VERSION.jar
+```
+
+1. Copy the `$METRON_HOME/bin/metron-rest` script to `/etc/init.d/metron-rest`
+
+### From Package Manager
+
+1. Deploy the RPM at `/incubator-metron/metron-deployment/packaging/docker/rpm-docker/target/RPMS/noarch/metron-rest-$METRON_VERSION-*.noarch.rpm`
+
+1. Install the RPM with:
+ ```
+ rpm -ih metron-rest-$METRON_VERSION-*.noarch.rpm
+ ```
+
+## Configuration
+
+The REST application depends on several configuration parameters:
+
+| Environment Variable | Description | Required | Default |
+| ------------------------------------- | ----------------------------------------------------------------- | -------- | ---------------------- |
+| METRON_USER | Run the application as this user | Optional | metron |
+| METRON_LOG_DIR | Directory where the log file is written | Optional | /var/log/metron/ |
+| METRON_PID_DIR | Directory where the pid file is written | Optional | /var/run/metron/ |
+| METRON_REST_PORT | REST application port | Optional | 8082 |
+| METRON_JVMFLAGS | JVM flags added to the start command | Optional | |
+| METRON_SPRING_PROFILES_ACTIVE | Active Spring profiles (see [below](#spring-profiles)) | Optional | |
+| METRON_JDBC_DRIVER | JDBC driver class | Required | |
+| METRON_JDBC_URL | JDBC url | Required | |
+| METRON_JDBC_USERNAME | JDBC username | Required | |
+| METRON_JDBC_PLATFORM | JDBC platform (one of h2, mysql, postgres, oracle | Required | |
+| METRON_JDBC_CLIENT_PATH | Path to JDBC client jar | Optional | H2 is bundled |
+| METRON_TEMP_GROK_PATH | Temporary directory used to test grok statements | Optional | ./patterns/temp |
+| METRON_DEFAULT_GROK_PATH | Defaults HDFS directory used to store grok statements | Optional | /apps/metron/patterns |
+| METRON_SPRING_OPTIONS | Additional Spring input parameters | Optional | |
+| ZOOKEEPER | Zookeeper quorum (ex. node1:2181,node2:2181) | Required | |
+| BROKERLIST | Kafka Broker list (ex. node1:6667,node2:6667) | Required | |
+| HDFS_URL | HDFS url or `fs.defaultFS` Hadoop setting (ex. hdfs://node1:8020) | Required | |
+| SECURITY_ENABLED | Enables Kerberos support | Optional | false |
+| METRON_PRINCIPAL_NAME | Kerberos principal for the metron user | Optional | |
+| METRON_SERVICE_KEYTAB | Path to the Kerberos keytab for the metron user | Optional | |
+
+These are set in the `/etc/sysconfig/metron` file.
+
+## Database setup
+
+The REST application persists data in a relational database and requires a dedicated database user and database (see https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html for more detail).
+
+### Development
+
+The REST application comes with embedded database support for development purposes (https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html#boot-features-embedded-database-support).
+
+For example, edit these variables in `/etc/sysconfig/metron` before starting the application to configure H2:
+```
+METRON_JDBC_DRIVER="org.h2.Driver"
+METRON_JDBC_URL="jdbc:h2:file:~/metrondb"
+METRON_JDBC_USERNAME="root"
+METRON_JDBC_PASSWORD='root"
+METRON_JDBC_PLATFORM="h2"
+```
+
+### Production
+
+The REST application should be configured with a production-grade database outside of development.
+
+For example, the following configures the application for MySQL:
+
+1. Install MySQL if not already available (this example uses version 5.7, installation instructions can be found [here](https://dev.mysql.com/doc/refman/5.7/en/linux-installation-yum-repo.html))
-1. Untar the archive in the target directory. The directory structure will look like:
- ```
- bin
- start_metron_rest.sh
- lib
- metron-rest-$METRON_VERSION.jar
- ```
+1. Create a metron user and REST database and permission the user for that database:
+```
+CREATE USER 'metron'@'node1' IDENTIFIED BY 'Myp@ssw0rd';
+CREATE DATABASE IF NOT EXISTS metronrest;
+GRANT ALL PRIVILEGES ON metronrest.* TO 'metron'@'node1';
+```
-1. Create an `application.yml` file with the contents of [application-vagrant.yml](src/main/resources/application-vagrant.yml). Substitute the appropriate Metron service hosts (Kafka, Zookeeper, Storm, etc.) in properties containing `node1` and update the `spring.datasource.*` properties as needed (see the [Security](#security) section for more details).
+1. Install the MySQL JDBC client onto the REST application host and configurate the METRON_JDBC_CLIENT_PATH variable:
+```
+cd $METRON_HOME/lib
+wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.41.tar.gz
+tar xf mysql-connector-java-5.1.41.tar.gz
+```
-1. Start the application with this command:
- ```
- ./bin/start_metron_rest.sh /path/to/application.yml
- ```
+1. Edit these variables in `/etc/sysconfig/metron` to configure the REST application for MySQL:
+```
+METRON_JDBC_DRIVER="com.mysql.jdbc.Driver"
+METRON_JDBC_URL="jdbc:mysql://mysql_host:3306/metronrest"
+METRON_JDBC_USERNAME="metron"
+METRON_JDBC_PLATFORM="mysql"
+METRON_JDBC_CLIENT_PATH=$METRON_HOME/lib/mysql-connector-java-5.1.41/mysql-connector-java-5.1.41-bin.jar
+```
## Usage
-The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port.
+After configuration is complete, the REST application can be managed as a service:
+```
+service metron-rest start
+```
+
+If a production database is configured, the JDBC password should be passed in as the first argument on startup:
+```
+service metron-rest start Myp@ssw0rd
+```
+
+The REST application can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8082.
## Security
-The metron-rest module uses [Spring Security](http://projects.spring.io/spring-security/) for authentication and stores user credentials in a relational database. The H2 database is configured by default and is intended only for development purposes. The "dev" profile can be used to automatically load test users:
+### Authentication
+
+The metron-rest module uses [Spring Security](http://projects.spring.io/spring-security/) for authentication and stores user credentials in the relational database configured above. The required tables are created automatically the first time the application is started so that should be done first. For example (continuing the MySQL example above), users can be added by connecting to MySQL and running:
```
-./bin/start_metron_rest.sh /path/to/application.yml --spring.profiles.active=dev
+use metronrest;
+insert into users (username, password, enabled) values ('your_username','your_password',1);
+insert into authorities (username, authority) values ('your_username', 'ROLE_USER');
```
-For [production use](http://docs.spring.io/spring-boot/docs/1.4.1.RELEASE/reference/htmlsingle/#boot-features-connect-to-production-database), a relational database should be configured. For example, configuring MySQL would be done as follows:
-
-1. Create a MySQL user for the Metron REST application (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html).
+### Kerberos
-1. Connect to MySQL and create a Metron REST database:
- ```
- CREATE DATABASE IF NOT EXISTS metronrest
- ```
+Metron REST can be configured for a cluster with Kerberos enabled. A client JAAS file is required for Kafka and Zookeeper and a Kerberos keytab for the metron user principal is required for all other services. Configure these settings in the `/etc/sysconfig/metron` file:
+```
+SECURITY_ENABLED=true
+METRON_JVMFLAGS="-Djava.security.auth.login.config=$METRON_HOME/client_jaas.conf"
+METRON_PRINCIPAL_NAME="metron@EXAMPLE.COM"
+METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab"
+```
-1. Add users:
- ```
- use metronrest;
- insert into users (username, password, enabled) values ('your_username','your_password',1);
- insert into authorities (username, authority) values ('your_username', 'ROLE_USER');
- ```
+## Spring Profiles
-1. Replace the H2 connection information in the application.yml file with MySQL connection information:
- ```
- spring:
- datasource:
- driverClassName: com.mysql.jdbc.Driver
- url: jdbc:mysql://mysql_host:3306/metronrest
- username: metron_rest_user
- password: metron_rest_password
- platform: mysql
- ```
+The REST application comes with a few [Spring Profiles](http://docs.spring.io/autorepo/docs/spring-boot/current/reference/html/boot-features-profiles.html) to aid in testing and development.
-1. Add a dependency for the MySQL JDBC connector in the metron-rest pom.xml:
- ```
- <dependency>
- <groupId>mysql</groupId>
- <artifactId>mysql-connector-java</artifactId>
- <version>${mysql.client.version}</version>
- </dependency>
- ```
+| Profile | Description |
+| ------------------------ | --------------------------------------------- |
+| test | sets variables to in-memory services, only used for integration testing |
+| dev | adds a test user to the database with credentials `user/password` |
+| vagrant | sets configuration variables to match the Metron vagrant environment |
+| docker | sets configuration variables to match the Metron dcoker environment |
-1. Follow the steps in the [Installation](#installation) section
+Setting active profiles is done with the METRON_SPRING_PROFILES_ACTIVE variable. For example, set this variable in `/etc/sysconfig/metron` to configure the REST application for the Vagrant environment and add a test user:
+```
+METRON_SPRING_PROFILES_ACTIVE="vagrant,dev"
+```
## API
@@ -100,7 +189,8 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| [ `DELETE /api/v1/kafka/topic/{name}`](#delete-apiv1kafkatopicname)|
| [ `GET /api/v1/kafka/topic/{name}/sample`](#get-apiv1kafkatopicnamesample)|
| [ `GET /api/v1/sensor/enrichment/config`](#get-apiv1sensorenrichmentconfig)|
-| [ `GET /api/v1/sensor/enrichment/config/list/available`](#get-apiv1sensorenrichmentconfiglistavailable)|
+| [ `GET /api/v1/sensor/enrichment/config/list/available/enrichments`](#get-apiv1sensorenrichmentconfiglistavailableenrichments)|
+| [ `GET /api/v1/sensor/enrichment/config/list/available/threat/triage/aggregators`](#get-apiv1sensorenrichmentconfiglistavailablethreattriageaggregators)|
| [ `DELETE /api/v1/sensor/enrichment/config/{name}`](#delete-apiv1sensorenrichmentconfigname)|
| [ `POST /api/v1/sensor/enrichment/config/{name}`](#post-apiv1sensorenrichmentconfigname)|
| [ `GET /api/v1/sensor/enrichment/config/{name}`](#get-apiv1sensorenrichmentconfigname)|
@@ -203,11 +293,11 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* 404 - File was not found in HDFS
### `GET /api/v1/hdfs/list`
- * Description: Reads a file from HDFS and returns the contents
+ * Description: Lists an HDFS directory
* Input:
* path - Path to HDFS directory
* Returns:
- * 200 - Returns file contents
+ * 200 - HDFS directory list
### `GET /api/v1/kafka/topic`
* Description: Retrieves all Kafka topics
@@ -250,11 +340,16 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* Returns:
* 200 - Returns all SensorEnrichmentConfigs
-### `GET /api/v1/sensor/enrichment/config/list/available`
+### `GET /api/v1/sensor/enrichment/config/list/available/enrichments`
* Description: Lists the available enrichments
* Returns:
* 200 - Returns a list of available enrichments
+### `GET /api/v1/sensor/enrichment/config/list/available/threat/triage/aggregators`
+ * Description: Lists the available threat triage aggregators
+ * Returns:
+ * 200 - Returns a list of available threat triage aggregators
+
### `DELETE /api/v1/sensor/enrichment/config/{name}`
* Description: Deletes a SensorEnrichmentConfig from Zookeeper
* Input:
@@ -515,38 +610,20 @@ mvn spring-boot:run -Drun.profiles=vagrant,dev
```
The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/.
-To run the application locally on the Quick Dev host, package the application and scp the archive to node1:
-```
-mvn clean package
-scp ./target/metron-rest-$METRON_VERSION-archive.tar.gz root@node1:$METRON_HOME
-```
-Login to node1 and unarchive the metron-rest application:
-```
-ssh root@node1
-cd $METRON_HOME && tar xf ./metron-rest-$METRON_VERSION-archive.tar.gz
-```
-Start the application on a different port to avoid conflicting with Ambari:
+To run the application locally on the Quick Dev host (node1), follow the [Installation](#installation) instructions above. Then set the METRON_SPRING_PROFILES_ACTIVE variable in `/etc/sysconfig/metron`:
```
-java -jar $METRON_HOME/lib/metron-rest-$METRON_VERSION.jar --spring.profiles.active=vagrant,dev --server.port=8082
+METRON_SPRING_PROFILES_ACTIVE="vagrant,dev"
```
-In a cluster with Kerberos enabled, first add metron-rest to the Kafka acls:
+and start the application:
```
-sudo su -
-export ZOOKEEPER=node1
-export BROKERLIST=node1
-export HDP_HOME="/usr/hdp/current"
-export METRON_VERSION="0.4.0"
-export METRON_HOME="/usr/metron/${METRON_VERSION}"
-${HDP_HOME}/kafka-broker/bin/kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=${ZOOKEEPER}:2181 --add --allow-principal User:metron --topic ambari_kafka_service_check
-${HDP_HOME}/kafka-broker/bin/kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=${ZOOKEEPER}:2181 --add --allow-principal User:metron --topic __consumer_offsets
-${HDP_HOME}/kafka-broker/bin/kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=${ZOOKEEPER}:2181 --add --allow-principal User:metron --group metron-rest
+service metron-rest start
```
-Then start the application as the metron user while including references to the jaas and krb5.confg files and enabling kerberos support:
+In a cluster with Kerberos enabled, update the security settings in `/etc/sysconfig/metron`. Security is disabled by default in the `vagrant` Spring profile so that setting must be overriden with the METRON_SPRING_OPTIONS variable:
```
-su metron
-cd ~
-java -Djava.security.auth.login.config=/home/metron/.storm/client_jaas.conf -Djava.security.krb5.conf=/etc/krb5.conf -jar $METRON_HOME/lib/metron-rest-$METRON_VERSION.jar --spring.profiles.active=vagrant,dev --server.port=8082 --kerberos.enabled=true
+METRON_SPRING_PROFILES_ACTIVE="vagrant,dev"
+METRON_JVMFLAGS="-Djava.security.auth.login.config=$METRON_HOME/client_jaas.conf"
+METRON_SPRING_OPTIONS="--kerberos.enabled=true"
```
The metron-rest application will be available at http://node1:8082/swagger-ui.html#/.
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-interface/metron-rest/src/main/config/rest_application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml b/metron-interface/metron-rest/src/main/config/rest_application.yml
new file mode 100644
index 0000000..dbb0b43
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/config/rest_application.yml
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+spring:
+ datasource:
+ driverClassName: ${METRON_JDBC_DRIVER}
+ url: ${METRON_JDBC_URL}
+ username: ${METRON_JDBC_USERNAME}
+ password: ${METRON_JDBC_PASSWORD}
+ platform: ${METRON_JDBC_PLATFORM}
+ continue-on-error: true
+
+zookeeper:
+ url: ${ZOOKEEPER}
+
+kafka:
+ broker:
+ url: ${BROKERLIST}
+
+hdfs:
+ namenode:
+ url: ${HDFS_URL}
+
+grok:
+ path:
+ temp: ${METRON_TEMP_GROK_PATH}
+ default: ${METRON_DEFAULT_GROK_PATH}
+
+storm:
+ ui:
+ url: ${STORM_REST_URL}
+ parser:
+ script.path: ${METRON_HOME}/bin/start_parser_topology.sh
+ enrichment:
+ script.path: ${METRON_HOME}/bin/start_enrichment_topology.sh
+ indexing:
+ script.path: ${METRON_HOME}/bin/start_elasticsearch_topology.sh
+
+kerberos:
+ enabled: ${SECURITY_ENABLED}
+ principal: ${METRON_PRINCIPAL_NAME}
+ keytab: ${METRON_SERVICE_KEYTAB}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
index c8ff0b6..b8957a9 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
@@ -44,8 +44,8 @@ public class HdfsController {
@Autowired
private HdfsService hdfsService;
- @ApiOperation(value = "Reads a file from HDFS and returns the contents")
- @ApiResponse(message = "Returns file contents", code = 200)
+ @ApiOperation(value = "Lists an HDFS directory")
+ @ApiResponse(message = "HDFS directory list", code = 200)
@RequestMapping(value = "/list", method = RequestMethod.GET)
ResponseEntity<List<String>> list(@ApiParam(name = "path", value = "Path to HDFS directory", required = true) @RequestParam String path) throws RestException {
return new ResponseEntity<>(hdfsService.list(new Path(path)), HttpStatus.OK);
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
index 158886f..c4964c0 100644
--- a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
@@ -16,7 +16,7 @@
spring:
datasource:
driverClassName: org.h2.Driver
- url: jdbc:h2:file:./metrondb
+ url: jdbc:h2:file:~/metrondb
username: root
password: root
platform: h2
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-interface/metron-rest/src/main/resources/schema-mysql.sql
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/schema-mysql.sql b/metron-interface/metron-rest/src/main/resources/schema-mysql.sql
index 986b652..638b7c5 100644
--- a/metron-interface/metron-rest/src/main/resources/schema-mysql.sql
+++ b/metron-interface/metron-rest/src/main/resources/schema-mysql.sql
@@ -14,9 +14,6 @@
See the License for the specific language governing permissions and
limitations under the License.
*/
-drop table if exists authorities;
-drop table if exists users;
-
create table if not exists users(
username varchar(50) not null primary key,
password varchar(50) not null,
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-interface/metron-rest/src/main/scripts/metron-rest
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest b/metron-interface/metron-rest/src/main/scripts/metron-rest
new file mode 100644
index 0000000..f9227a9
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/scripts/metron-rest
@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# metron rest application service
+# chkconfig: - 20 80
+# description: rest application
+# processname: metron-rest
+#
+if [ -f /etc/rc.d/init.d/functions ]; then
+ . /etc/rc.d/init.d/functions
+fi
+
+NAME=metron-rest
+DESC="Metron REST Application"
+METRON_VERSION=${project.version}
+METRON_HOME=/usr/metron/$METRON_VERSION
+METRON_LOG_DIR="/var/log/metron"
+METRON_PID_DIR="/var/run/metron"
+METRON_USER="metron"
+METRON_GROUP="metron"
+METRON_REST_PORT=8082
+METRON_JDBC_PASSWORD="$2"
+METRON_SYSCONFIG="/etc/sysconfig/metron"
+if [ -f "$METRON_SYSCONFIG" ]; then
+ set -a
+ . "$METRON_SYSCONFIG"
+fi
+
+PIDFILE="$METRON_PID_DIR/$NAME.pid"
+LOCKFILE=/var/lock/subsys/$NAME
+
+# the vagrant Spring profile provides configuration values, otherwise configuration is provided by rest_application.yml
+if [[ !($METRON_SPRING_PROFILES_ACTIVE == *"vagrant"*) ]]; then
+ METRON_SPRING_OPTIONS+=" --spring.config.location=$METRON_HOME/config/rest_application.yml"
+fi
+METRON_SPRING_OPTIONS+=" --server.port=$METRON_REST_PORT"
+if [ $METRON_SPRING_PROFILES_ACTIVE ]; then
+ METRON_SPRING_OPTIONS+=" --spring.profiles.active=$METRON_SPRING_PROFILES_ACTIVE"
+fi
+if [ $METRON_JDBC_PASSWORD ]; then
+ METRON_SPRING_OPTIONS+=" --spring.datasource.password=$METRON_JDBC_PASSWORD"
+fi
+if [ $METRON_JDBC_CLIENT_PATH ]; then
+ METRON_JVMFLAGS+=" -Dloader.path=$METRON_JDBC_CLIENT_PATH"
+fi
+DAEMON="java $METRON_JVMFLAGS -jar $METRON_HOME/lib/metron-rest-$METRON_VERSION.jar $METRON_SPRING_OPTIONS"
+
+#
+# start the rest application
+#
+start() {
+
+ # if pidfile exists, do not start another
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ printf "OK [$PID]\n"
+ return
+ fi
+
+ if [ ! -d "$METRON_LOG_DIR" ]; then
+ mkdir -p "$METRON_LOG_DIR" && chown "$METRON_USER":"$METRON_GROUP" "$METRON_LOG_DIR"
+ fi
+
+ if [ ! -d "$METRON_PID_DIR" ]; then
+ mkdir -p "$METRON_PID_DIR" && chown "$METRON_USER":"$METRON_GROUP" "$METRON_PID_DIR"
+ fi
+
+ # kick-off the daemon
+ PID=`runuser -s /bin/bash $METRON_USER -c "$DAEMON" >> $METRON_LOG_DIR/$NAME.log 2>&1 & echo $!`
+
+ if [ -z $PID ]; then
+ printf "Fail\n"
+ else
+ echo $PID > $PIDFILE && touch $LOCKFILE
+ printf "Ok [$PID]\n"
+ fi
+}
+
+#
+# stop the rest application
+#
+stop() {
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ killproc -p $PIDFILE metron-rest
+ RETVAL=$?
+ echo
+ [ $RETVAL -eq 0 ] && rm -f $LOCKFILE
+ printf "Stopped [$PID]\n"
+ else
+ printf "Not running\n"
+ fi
+}
+
+#
+# status check of the rest application
+#
+rest_status() {
+ status -p $PIDFILE metron-rest
+}
+
+case "$1" in
+
+ ##############################################################################
+ # start
+ #
+ start)
+ printf "%-50s \n" "Starting $NAME..."
+ start
+ ;;
+
+ ##############################################################################
+ # status
+ #
+ status)
+ printf "%-50s \n" "Checking $NAME..."
+ rest_status
+ ;;
+
+ ##############################################################################
+ # stop
+ #
+ stop)
+ printf "%-50s \n" "Stopping $NAME..."
+ stop
+ ;;
+
+ ##############################################################################
+ # restart
+ #
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ ##############################################################################
+ # reload
+ #
+ reload)
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh b/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh
deleted file mode 100755
index ea5d6c9..0000000
--- a/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-METRON_VERSION=${project.version}
-SCRIPTS_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-if [ "$#" -lt 1 ]; then
- echo "Usage: start.sh <path to application.yml file> <spring options>"
- echo "Path can be absolute or relative to $SCRIPTS_ROOT"
-else
- java -jar $SCRIPTS_ROOT/../lib/metron-rest-$METRON_VERSION.jar --spring.config.location=$@
-fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/716bda32/metron-interface/metron-rest/src/test/resources/README.vm
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/resources/README.vm b/metron-interface/metron-rest/src/test/resources/README.vm
index cff7ca7..2d8fd75 100644
--- a/metron-interface/metron-rest/src/test/resources/README.vm
+++ b/metron-interface/metron-rest/src/test/resources/README.vm
@@ -7,76 +7,165 @@ This module provides a RESTful API for interacting with Metron.
* A running Metron cluster
* Java 8 installed
* Storm CLI and Metron topology scripts (start_parser_topology.sh, start_enrichment_topology.sh, start_elasticsearch_topology.sh) installed
+* A relational database
#[[##]]# Installation
+
+#[[###]]# From Source
+
1. Package the application with Maven:
- ```
- mvn clean package
- ```
+```
+mvn clean package
+```
+
+1. Untar the archive in the $METRON_HOME directory. The directory structure will look like:
+```
+config
+ rest_application.yml
+bin
+ metron-rest
+lib
+ metron-rest-$METRON_VERSION.jar
+```
+
+1. Copy the `$METRON_HOME/bin/metron-rest` script to `/etc/init.d/metron-rest`
+
+#[[###]]# From Package Manager
+
+1. Deploy the RPM at `/incubator-metron/metron-deployment/packaging/docker/rpm-docker/target/RPMS/noarch/metron-rest-$METRON_VERSION-*.noarch.rpm`
+
+1. Install the RPM with:
+ ```
+ rpm -ih metron-rest-$METRON_VERSION-*.noarch.rpm
+ ```
+
+#[[##]]# Configuration
+
+The REST application depends on several configuration parameters:
+
+| Environment Variable | Description | Required | Default |
+| ------------------------------------- | ----------------------------------------------------------------- | -------- | ---------------------- |
+| METRON_USER | Run the application as this user | Optional | metron |
+| METRON_LOG_DIR | Directory where the log file is written | Optional | /var/log/metron/ |
+| METRON_PID_DIR | Directory where the pid file is written | Optional | /var/run/metron/ |
+| METRON_REST_PORT | REST application port | Optional | 8082 |
+| METRON_JVMFLAGS | JVM flags added to the start command | Optional | |
+| METRON_SPRING_PROFILES_ACTIVE | Active Spring profiles (see [below](#spring-profiles)) | Optional | |
+| METRON_JDBC_DRIVER | JDBC driver class | Required | |
+| METRON_JDBC_URL | JDBC url | Required | |
+| METRON_JDBC_USERNAME | JDBC username | Required | |
+| METRON_JDBC_PLATFORM | JDBC platform (one of h2, mysql, postgres, oracle | Required | |
+| METRON_JDBC_CLIENT_PATH | Path to JDBC client jar | Optional | H2 is bundled |
+| METRON_TEMP_GROK_PATH | Temporary directory used to test grok statements | Optional | ./patterns/temp |
+| METRON_DEFAULT_GROK_PATH | Defaults HDFS directory used to store grok statements | Optional | /apps/metron/patterns |
+| METRON_SPRING_OPTIONS | Additional Spring input parameters | Optional | |
+| ZOOKEEPER | Zookeeper quorum (ex. node1:2181,node2:2181) | Required | |
+| BROKERLIST | Kafka Broker list (ex. node1:6667,node2:6667) | Required | |
+| HDFS_URL | HDFS url or `fs.defaultFS` Hadoop setting (ex. hdfs://node1:8020) | Required | |
+| SECURITY_ENABLED | Enables Kerberos support | Optional | false |
+| METRON_PRINCIPAL_NAME | Kerberos principal for the metron user | Optional | |
+| METRON_SERVICE_KEYTAB | Path to the Kerberos keytab for the metron user | Optional | |
+
+These are set in the `/etc/sysconfig/metron` file.
+
+#[[##]]# Database setup
+
+The REST application persists data in a relational database and requires a dedicated database user and database (see https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html for more detail).
+
+#[[###]]# Development
+
+The REST application comes with embedded database support for development purposes (https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html#boot-features-embedded-database-support).
+
+For example, edit these variables in `/etc/sysconfig/metron` before starting the application to configure H2:
+```
+METRON_JDBC_DRIVER="org.h2.Driver"
+METRON_JDBC_URL="jdbc:h2:file:~/metrondb"
+METRON_JDBC_USERNAME="root"
+METRON_JDBC_PASSWORD='root"
+METRON_JDBC_PLATFORM="h2"
+```
+
+#[[###]]# Production
+
+The REST application should be configured with a production-grade database outside of development.
-1. Untar the archive in the target directory. The directory structure will look like:
- ```
- bin
- start_metron_rest.sh
- lib
- metron-rest-$METRON_VERSION.jar
- ```
+For example, the following configures the application for MySQL:
-1. Create an `application.yml` file with the contents of [application-vagrant.yml](src/main/resources/application-vagrant.yml). Substitute the appropriate Metron service hosts (Kafka, Zookeeper, Storm, etc.) in properties containing `node1` and update the `spring.datasource.*` properties as needed (see the [Security](#security) section for more details).
+1. Install MySQL if not already available (this example uses version 5.7, installation instructions can be found [here](https://dev.mysql.com/doc/refman/5.7/en/linux-installation-yum-repo.html))
-1. Start the application with this command:
- ```
- ./bin/start_metron_rest.sh /path/to/application.yml
- ```
+1. Create a metron user and REST database and permission the user for that database:
+```
+CREATE USER 'metron'@'node1' IDENTIFIED BY 'Myp@ssw0rd';
+CREATE DATABASE IF NOT EXISTS metronrest;
+GRANT ALL PRIVILEGES ON metronrest.* TO 'metron'@'node1';
+```
+
+1. Install the MySQL JDBC client onto the REST application host and configurate the METRON_JDBC_CLIENT_PATH variable:
+```
+cd $METRON_HOME/lib
+wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.41.tar.gz
+tar xf mysql-connector-java-5.1.41.tar.gz
+```
+
+1. Edit these variables in `/etc/sysconfig/metron` to configure the REST application for MySQL:
+```
+METRON_JDBC_DRIVER="com.mysql.jdbc.Driver"
+METRON_JDBC_URL="jdbc:mysql://mysql_host:3306/metronrest"
+METRON_JDBC_USERNAME="metron"
+METRON_JDBC_PLATFORM="mysql"
+METRON_JDBC_CLIENT_PATH=$METRON_HOME/lib/mysql-connector-java-5.1.41/mysql-connector-java-5.1.41-bin.jar
+```
#[[##]]# Usage
-The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port.
+After configuration is complete, the REST application can be managed as a service:
+```
+service metron-rest start
+```
+
+If a production database is configured, the JDBC password should be passed in as the first argument on startup:
+```
+service metron-rest start Myp@ssw0rd
+```
+
+The REST application can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8082.
#[[##]]# Security
-The metron-rest module uses [Spring Security](http://projects.spring.io/spring-security/) for authentication and stores user credentials in a relational database. The H2 database is configured by default and is intended only for development purposes. The "dev" profile can be used to automatically load test users:
+#[[###]]# Authentication
+
+The metron-rest module uses [Spring Security](http://projects.spring.io/spring-security/) for authentication and stores user credentials in the relational database configured above. The required tables are created automatically the first time the application is started so that should be done first. For example (continuing the MySQL example above), users can be added by connecting to MySQL and running:
```
-./bin/start_metron_rest.sh /path/to/application.yml --spring.profiles.active=dev
+use metronrest;
+insert into users (username, password, enabled) values ('your_username','your_password',1);
+insert into authorities (username, authority) values ('your_username', 'ROLE_USER');
```
-For [production use](http://docs.spring.io/spring-boot/docs/1.4.1.RELEASE/reference/htmlsingle/#boot-features-connect-to-production-database), a relational database should be configured. For example, configuring MySQL would be done as follows:
-
-1. Create a MySQL user for the Metron REST application (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html).
+#[[###]]# Kerberos
-1. Connect to MySQL and create a Metron REST database:
- ```
- CREATE DATABASE IF NOT EXISTS metronrest
- ```
+Metron REST can be configured for a cluster with Kerberos enabled. A client JAAS file is required for Kafka and Zookeeper and a Kerberos keytab for the metron user principal is required for all other services. Configure these settings in the `/etc/sysconfig/metron` file:
+```
+SECURITY_ENABLED=true
+METRON_JVMFLAGS="-Djava.security.auth.login.config=$METRON_HOME/client_jaas.conf"
+METRON_PRINCIPAL_NAME="metron@EXAMPLE.COM"
+METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab"
+```
-1. Add users:
- ```
- use metronrest;
- insert into users (username, password, enabled) values ('your_username','your_password',1);
- insert into authorities (username, authority) values ('your_username', 'ROLE_USER');
- ```
+#[[##]]# Spring Profiles
-1. Replace the H2 connection information in the application.yml file with MySQL connection information:
- ```
- spring:
- datasource:
- driverClassName: com.mysql.jdbc.Driver
- url: jdbc:mysql://mysql_host:3306/metronrest
- username: metron_rest_user
- password: metron_rest_password
- platform: mysql
- ```
+The REST application comes with a few [Spring Profiles](http://docs.spring.io/autorepo/docs/spring-boot/current/reference/html/boot-features-profiles.html) to aid in testing and development.
-1. Add a dependency for the MySQL JDBC connector in the metron-rest pom.xml:
- ```
- <dependency>
- <groupId>mysql</groupId>
- <artifactId>mysql-connector-java</artifactId>
- <version>${mysql.client.version}</version>
- </dependency>
- ```
+| Profile | Description |
+| ------------------------ | --------------------------------------------- |
+| test | sets variables to in-memory services, only used for integration testing |
+| dev | adds a test user to the database with credentials `user/password` |
+| vagrant | sets configuration variables to match the Metron vagrant environment |
+| docker | sets configuration variables to match the Metron dcoker environment |
-1. Follow the steps in the [Installation](#installation) section
+Setting active profiles is done with the METRON_SPRING_PROFILES_ACTIVE variable. For example, set this variable in `/etc/sysconfig/metron` to configure the REST application for the Vagrant environment and add a test user:
+```
+METRON_SPRING_PROFILES_ACTIVE="vagrant,dev"
+```
#[[##]]# API
@@ -125,38 +214,20 @@ mvn spring-boot:run -Drun.profiles=vagrant,dev
```
The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/.
-To run the application locally on the Quick Dev host, package the application and scp the archive to node1:
-```
-mvn clean package
-scp ./target/metron-rest-$METRON_VERSION-archive.tar.gz root@node1:$METRON_HOME
-```
-Login to node1 and unarchive the metron-rest application:
-```
-ssh root@node1
-cd $METRON_HOME && tar xf ./metron-rest-$METRON_VERSION-archive.tar.gz
-```
-Start the application on a different port to avoid conflicting with Ambari:
+To run the application locally on the Quick Dev host (node1), follow the [Installation](#installation) instructions above. Then set the METRON_SPRING_PROFILES_ACTIVE variable in `/etc/sysconfig/metron`:
```
-java -jar $METRON_HOME/lib/metron-rest-$METRON_VERSION.jar --spring.profiles.active=vagrant,dev --server.port=8082
+METRON_SPRING_PROFILES_ACTIVE="vagrant,dev"
```
-In a cluster with Kerberos enabled, first add metron-rest to the Kafka acls:
+and start the application:
```
-sudo su -
-export ZOOKEEPER=node1
-export BROKERLIST=node1
-export HDP_HOME="/usr/hdp/current"
-export METRON_VERSION="0.4.0"
-export METRON_HOME="/usr/metron/${METRON_VERSION}"
-${HDP_HOME}/kafka-broker/bin/kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=${ZOOKEEPER}:2181 --add --allow-principal User:metron --topic ambari_kafka_service_check
-${HDP_HOME}/kafka-broker/bin/kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=${ZOOKEEPER}:2181 --add --allow-principal User:metron --topic __consumer_offsets
-${HDP_HOME}/kafka-broker/bin/kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=${ZOOKEEPER}:2181 --add --allow-principal User:metron --group metron-rest
+service metron-rest start
```
-Then start the application as the metron user while including references to the jaas and krb5.confg files and enabling kerberos support:
+In a cluster with Kerberos enabled, update the security settings in `/etc/sysconfig/metron`. Security is disabled by default in the `vagrant` Spring profile so that setting must be overriden with the METRON_SPRING_OPTIONS variable:
```
-su metron
-cd ~
-java -Djava.security.auth.login.config=/home/metron/.storm/client_jaas.conf -Djava.security.krb5.conf=/etc/krb5.conf -jar $METRON_HOME/lib/metron-rest-$METRON_VERSION.jar --spring.profiles.active=vagrant,dev --server.port=8082 --kerberos.enabled=true
+METRON_SPRING_PROFILES_ACTIVE="vagrant,dev"
+METRON_JVMFLAGS="-Djava.security.auth.login.config=$METRON_HOME/client_jaas.conf"
+METRON_SPRING_OPTIONS="--kerberos.enabled=true"
```
The metron-rest application will be available at http://node1:8082/swagger-ui.html#/.