You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2019/04/12 18:52:15 UTC
[metron] branch master updated: METRON-2050 Automatically populate
a list of enrichments from HBase (mmiklavc) closes apache/metron#1365
This is an automated email from the ASF dual-hosted git repository.
mmiklavcic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 5709548 METRON-2050 Automatically populate a list of enrichments from HBase (mmiklavc) closes apache/metron#1365
5709548 is described below
commit 57095488f28a017e290e021fea65412e7240e8da
Author: mmiklavc <mi...@gmail.com>
AuthorDate: Fri Apr 12 12:50:00 2019 -0600
METRON-2050 Automatically populate a list of enrichments from HBase (mmiklavc) closes apache/metron#1365
---
metron-deployment/packaging/ambari/README.md | 2 +-
.../configuration/metron-enrichment-env.xml | 24 ++
.../common-services/METRON/CURRENT/metainfo.xml | 4 +-
.../CURRENT/package/scripts/enrichment_commands.py | 117 ++++++--
.../CURRENT/package/scripts/enrichment_master.py | 6 +-
.../CURRENT/package/scripts/metron_service.py | 16 +-
.../CURRENT/package/scripts/params/params_linux.py | 20 +-
.../package/scripts/params/status_params.py | 13 +-
.../packaging/docker/rpm-docker/SPECS/metron.spec | 23 ++
.../packaging/docker/rpm-docker/pom.xml | 6 +
metron-interface/metron-rest/README.md | 2 +-
metron-interface/metron-rest/pom.xml | 19 ++
.../org/apache/metron/rest/config/HBaseConfig.java | 33 ++-
.../service/SensorEnrichmentConfigService.java | 8 +-
.../rest/service/impl/AlertsUIServiceImpl.java | 23 +-
.../impl/SensorEnrichmentConfigServiceImpl.java | 40 ++-
.../metron/rest/user}/UserSettingsClient.java | 19 +-
.../apache/metron/rest/config/HBaseConfigTest.java | 23 +-
.../org/apache/metron/rest/config/TestConfig.java | 29 +-
...rEnrichmentConfigControllerIntegrationTest.java | 33 ++-
.../rest/service/impl/AlertsUIServiceImplTest.java | 11 +-
.../SensorEnrichmentConfigServiceImplTest.java | 43 +--
.../metron/rest/user}/UserSettingsClientTest.java | 30 +-
.../src/test/resources/zookeeper/global.json | 2 +-
metron-platform/metron-common/README.md | 73 ++---
.../configuration/EnrichmentConfigurations.java | 5 +
metron-platform/metron-data-management/pom.xml | 12 +
.../metron/dataloads/hbase/mr/HBaseUtil.java | 54 ++--
metron-platform/metron-enrichment/README.md | 14 +
metron-platform/metron-hbase-server/README.md | 44 +++
metron-platform/metron-hbase-server/pom.xml | 326 +++++++++++++++++++++
.../src/main/assembly/assembly.xml | 54 ++++
.../hbase/coprocessor/EnrichmentCoprocessor.java | 198 +++++++++++++
.../hbase/coprocessor/GlobalConfigService.java} | 16 +-
.../metron/hbase/coprocessor/HBaseCacheWriter.java | 78 +++++
.../main/scripts/load_enrichment_coprocessor.sh | 38 +++
.../EnrichmentCoprocessorIntegrationTest.java | 208 +++++++++++++
.../coprocessor/EnrichmentCoprocessorTest.java | 167 +++++++++++
.../org/apache/metron/hbase/helper/HelperDao.java | 58 ++++
.../org/apache/metron/hbase/HTableProvider.java | 3 +-
.../org/apache/metron/hbase/TableProvider.java | 8 +
.../apache/metron/hbase/client/HBaseClient.java | 34 +++
.../src/test/resources/log4j.properties | 7 +-
.../apache/metron/test/utils/UnitTestHelper.java | 31 +-
metron-platform/pom.xml | 1 +
45 files changed, 1761 insertions(+), 214 deletions(-)
diff --git a/metron-deployment/packaging/ambari/README.md b/metron-deployment/packaging/ambari/README.md
index 14dd5ca..4b0aace 100644
--- a/metron-deployment/packaging/ambari/README.md
+++ b/metron-deployment/packaging/ambari/README.md
@@ -61,7 +61,7 @@ The layout of `/common-services/METRON/CURRENT` is
* `kerberos.json`
* Defines the keytabs and other Kerberos configuration to be used when Kerberizing a cluster
* `metainfo.xml`
- * Defines the METRON service, along with required packages, services, etc.
+ * Defines the METRON service, along with required packages, services, etc. If you need to have the MPack install a new package (e.g. RPM, DEB), add it here.
* `service_advisor.py`
* Handles component layout and validation, along with handling some configurations for other services or that needs configs from other services.
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
index 950db6a..1fd4702 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
@@ -135,6 +135,30 @@
<description>The HBase column family which will hold threatintel data in HBase.</description>
<display-name>HBase Table Column Family</display-name>
</property>
+ <property>
+ <name>enrichment_list_hbase_provider_impl</name>
+ <value>org.apache.metron.hbase.HTableProvider</value>
+ <description>The HBase table provider implementation to use for obtaining table references.</description>
+ <display-name>Enrichment List HBase Table Provider Implementation</display-name>
+ </property>
+ <property>
+ <name>enrichment_list_hbase_coprocessor_impl</name>
+ <value>org.apache.metron.hbase.coprocessor.EnrichmentCoprocessor</value>
+ <description>The HBase coprocessor implementation to use for managing the enrichment list.</description>
+ <display-name>Enrichment List HBase Coprocessor Implementation</display-name>
+ </property>
+ <property>
+ <name>enrichment_list_hbase_table</name>
+ <value>enrichment_list</value>
+ <description>The HBase table which will hold the list of enrichment types gathered by the enrichment coprocessor.</description>
+ <display-name>Enrichment List HBase Table</display-name>
+ </property>
+ <property>
+ <name>enrichment_list_hbase_cf</name>
+ <value>t</value>
+ <description>The HBase column family which will hold enrichment list data in HBase.</description>
+ <display-name>Enrichment List HBase Column Family</display-name>
+ </property>
<!--
storm common parameters
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 5112868..ddc56cf 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -441,6 +441,9 @@
<name>metron-data-management</name>
</package>
<package>
+ <name>metron-hbase-server</name>
+ </package>
+ <package>
<name>metron-management</name>
</package>
<package>
@@ -497,7 +500,6 @@
<package>
<name>metron-performance</name>
</package>
-
</packages>
</osSpecific>
<osSpecific>
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
index 1b71e60..5d82c8c 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
@@ -46,6 +46,7 @@ class EnrichmentCommands:
self.__kafka_configured = os.path.isfile(self.__params.enrichment_kafka_configured_flag_file)
self.__kafka_acl_configured = os.path.isfile(self.__params.enrichment_kafka_acl_configured_flag_file)
self.__hbase_configured = os.path.isfile(self.__params.enrichment_hbase_configured_flag_file)
+ self.__hbase_coprocessor_configured = os.path.isfile(self.__params.enrichment_hbase_coprocessor_configured_flag_file)
self.__hbase_acl_configured = os.path.isfile(self.__params.enrichment_hbase_acl_configured_flag_file)
self.__maxmind_configured = os.path.isfile(self.__params.enrichment_maxmind_configured_flag_file)
@@ -64,6 +65,9 @@ class EnrichmentCommands:
def is_hbase_configured(self):
return self.__hbase_configured
+ def is_hbase_coprocessor_configured(self):
+ return self.__hbase_coprocessor_configured
+
def is_hbase_acl_configured(self):
return self.__hbase_acl_configured
@@ -79,6 +83,9 @@ class EnrichmentCommands:
def set_hbase_configured(self):
metron_service.set_configured(self.__params.metron_user, self.__params.enrichment_hbase_configured_flag_file, "Setting HBase configured to True for enrichment")
+ def set_hbase_coprocessor_configured(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.enrichment_hbase_coprocessor_configured_flag_file, "Setting HBase coprocessor configured to True for enrichment")
+
def set_hbase_acl_configured(self):
metron_service.set_configured(self.__params.metron_user, self.__params.enrichment_hbase_acl_configured_flag_file, "Setting HBase ACL configured to True for enrichment")
@@ -206,14 +213,64 @@ class EnrichmentCommands:
def create_hbase_tables(self):
Logger.info("Creating HBase Tables")
metron_service.create_hbase_table(self.__params,
- self.__params.enrichment_hbase_table,
- self.__params.enrichment_hbase_cf)
+ self.__params.enrichment_hbase_table,
+ self.__params.enrichment_hbase_cf)
+ metron_service.create_hbase_table(self.__params,
+ self.__params.enrichment_list_hbase_table,
+ self.__params.enrichment_list_hbase_cf)
metron_service.create_hbase_table(self.__params,
self.__params.threatintel_hbase_table,
self.__params.threatintel_hbase_cf)
Logger.info("Done creating HBase Tables")
self.set_hbase_configured()
+ def load_enrichment_coprocessor(self):
+ Logger.info("Creating HDFS location for enrichment coprocessor and loading from local disk")
+
+ self.__params.HdfsResource(self.__params.hbase_coprocessor_hdfs_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=self.__params.metron_user,
+ group=self.__params.metron_group,
+ mode=0755,
+ source=self.__params.hbase_coprocessor_local_dir,
+ recursive_chown = True)
+
+ Logger.info("Loading HBase coprocessor for enrichments")
+ Logger.info("See https://hbase.apache.org/1.1/book.html#load_coprocessor_in_shell for more detail")
+
+ Logger.info("HBase coprocessor setup - first disabling the enrichments HBase table.")
+ command_template = "echo \"disable '{0}'\" | hbase shell -n"
+ command = command_template.format(self.__params.enrichment_hbase_table)
+ Logger.info("Executing command " + command)
+ Execute(command, user=self.__params.metron_user, tries=1, logoutput=True)
+
+ Logger.info("HBase coprocessor setup - altering table and adding coprocessor.")
+ command_template = "{0}/bin/load_enrichment_coprocessor.sh {1} {2} {3} {4} {5}"
+ command = command_template.format(self.__params.metron_home,
+ self.__params.enrichment_hbase_table,
+ self.__params.hdfs_url,
+ self.__params.hbase_coprocessor_hdfs_dir,
+ self.__params.enrichment_list_hbase_coprocessor_impl,
+ self.__params.zookeeper_quorum)
+ Logger.info("Executing command " + command)
+ Execute(command, user=self.__params.metron_user, tries=1, logoutput=True)
+
+ Logger.info("HBase coprocessor setup - re-enabling enrichments table.")
+ command_template = "echo \"enable'{0}'\" | hbase shell -n"
+ command = command_template.format(self.__params.enrichment_hbase_table)
+ Logger.info("Executing command " + command)
+ Execute(command, user=self.__params.metron_user, tries=1, logoutput=True)
+
+ Logger.info("HBase coprocessor setup - verifying coprocessor was loaded. The coprocessor should be listed in the TABLE_ATTRIBUTES.")
+ command_template = "echo \"describe '{0}'\" | hbase shell -n"
+ command = command_template.format(self.__params.enrichment_hbase_table)
+ Logger.info("Executing command " + command)
+ Execute(command, user=self.__params.metron_user, tries=1, logoutput=True)
+
+ Logger.info("Done loading HBase coprocessor for enrichments")
+ self.set_hbase_coprocessor_configured()
+
def set_hbase_acls(self):
Logger.info("Setting HBase ACLs")
if self.__params.security_enabled:
@@ -232,6 +289,15 @@ class EnrichmentCommands:
user=self.__params.hbase_user
)
+ add_enrichment_list_acl_cmd = cmd.format(self.__params.metron_user, self.__params.enrichment_list_hbase_table)
+ Execute(add_enrichment_list_acl_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ user=self.__params.hbase_user
+ )
+
add_threatintel_acl_cmd = cmd.format(self.__params.metron_user, self.__params.threatintel_hbase_table)
Execute(add_threatintel_acl_cmd,
tries=3,
@@ -260,31 +326,42 @@ class EnrichmentCommands:
Logger.info("Checking HBase for Enrichment")
metron_service.check_hbase_table(
- self.__params,
- self.__params.enrichment_hbase_table)
+ self.__params,
+ self.__params.enrichment_hbase_table)
metron_service.check_hbase_column_family(
- self.__params,
- self.__params.enrichment_hbase_table,
- self.__params.enrichment_hbase_cf)
+ self.__params,
+ self.__params.enrichment_hbase_table,
+ self.__params.enrichment_hbase_cf)
+
+ Logger.info("Checking HBase for Enrichment List")
+ metron_service.check_hbase_table(
+ self.__params,
+ self.__params.enrichment_list_hbase_table)
+ metron_service.check_hbase_column_family(
+ self.__params,
+ self.__params.enrichment_list_hbase_table,
+ self.__params.enrichment_list_hbase_cf)
Logger.info("Checking HBase for Threat Intel")
metron_service.check_hbase_table(
- self.__params,
- self.__params.threatintel_hbase_table)
+ self.__params,
+ self.__params.threatintel_hbase_table)
metron_service.check_hbase_column_family(
- self.__params,
- self.__params.threatintel_hbase_table,
- self.__params.threatintel_hbase_cf)
+ self.__params,
+ self.__params.threatintel_hbase_table,
+ self.__params.threatintel_hbase_cf)
if self.__params.security_enabled:
-
- Logger.info('Checking Kafka ACLs for Enrichment')
- metron_service.check_kafka_acls(self.__params, self.__get_topics())
- metron_service.check_kafka_acl_groups(self.__params, self.__get_kafka_acl_groups())
-
- Logger.info("Checking HBase ACLs for Enrichment")
- metron_service.check_hbase_acls(self.__params, self.__params.enrichment_hbase_table)
- metron_service.check_hbase_acls(self.__params, self.__params.threatintel_hbase_table)
+ Logger.info('Checking Kafka ACLs for Enrichment')
+ metron_service.check_kafka_acls(self.__params, self.__get_topics())
+ metron_service.check_kafka_acl_groups(self.__params, self.__get_kafka_acl_groups())
+
+ Logger.info("Checking HBase ACLs for Enrichment")
+ metron_service.check_hbase_acls(self.__params, self.__params.enrichment_hbase_table)
+ Logger.info("Checking HBase ACLs for Enrichment List")
+ metron_service.check_hbase_acls(self.__params, self.__params.enrichment_list_hbase_table)
+ Logger.info("Checking HBase ACLs for Threat Intel")
+ metron_service.check_hbase_acls(self.__params, self.__params.threatintel_hbase_table)
Logger.info("Checking for Enrichment topology")
if not self.is_topology_active(env):
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
index 3267c9b..ff89b6b 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
@@ -49,8 +49,8 @@ class Enrichment(Script):
group=params.metron_group)
if not metron_service.is_zk_configured(params):
- metron_service.init_zk_config(params)
- metron_service.set_zk_configured(params)
+ metron_service.init_zk_config(params)
+ metron_service.set_zk_configured(params)
metron_service.refresh_configs(params)
Logger.info("Calling security setup")
@@ -74,6 +74,8 @@ class Enrichment(Script):
commands.init_kafka_acls()
if not commands.is_hbase_configured():
commands.create_hbase_tables()
+ if not commands.is_hbase_coprocessor_configured():
+ commands.load_enrichment_coprocessor()
if params.security_enabled and not commands.is_hbase_acl_configured():
commands.set_hbase_acls()
if not commands.is_maxmind_configured():
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
index 5b35e64..d7c505e 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
@@ -117,6 +117,21 @@ def build_global_config_patch(params, patch_file):
},
{
"op": "add",
+ "path": "/enrichment.list.hbase.provider.impl",
+ "value": "{{enrichment_list_hbase_provider_impl}}"
+ },
+ {
+ "op": "add",
+ "path": "/enrichment.list.hbase.table",
+ "value": "{{enrichment_list_hbase_table}}"
+ },
+ {
+ "op": "add",
+ "path": "/enrichment.list.hbase.cf",
+ "value": "{{enrichment_list_hbase_cf}}"
+ },
+ {
+ "op": "add",
"path": "/update.hbase.table",
"value": "{{update_hbase_table}}"
},
@@ -384,7 +399,6 @@ def create_hbase_table(params, table, cf):
user=params.hbase_user
)
-
def check_hbase_table(params, table):
"""
Validates that an HBase table exists. An exception is raised if the table
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 095acda..64105e3 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -43,11 +43,14 @@ hdp_version = default("/commandParams/version", None)
hostname = config['hostname']
metron_home = status_params.metron_home
+metron_apps_hdfs_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
parsers = status_params.parsers
parser_error_topic = config['configurations']['metron-parsers-env']['parser_error_topic']
-geoip_hdfs_dir = "/apps/metron/geo/default/"
-asn_hdfs_dir = "/apps/metron/asn/default/"
+geoip_hdfs_dir = metron_apps_hdfs_dir + "/geo/default/"
+asn_hdfs_dir = metron_apps_hdfs_dir + "/asn/default/"
+hbase_coprocessor_local_dir = format("{metron_home}/coprocessor")
+hbase_coprocessor_hdfs_dir = metron_apps_hdfs_dir + "/coprocessor"
metron_user = status_params.metron_user
metron_group = config['configurations']['metron-env']['metron_group']
metron_log_dir = config['configurations']['metron-env']['metron_log_dir']
@@ -89,6 +92,7 @@ parsers_acl_configured_flag_file = status_params.parsers_acl_configured_flag_fil
enrichment_kafka_configured_flag_file = status_params.enrichment_kafka_configured_flag_file
enrichment_kafka_acl_configured_flag_file = status_params.enrichment_kafka_acl_configured_flag_file
enrichment_hbase_configured_flag_file = status_params.enrichment_hbase_configured_flag_file
+enrichment_hbase_coprocessor_configured_flag_file = status_params.enrichment_hbase_coprocessor_configured_flag_file
enrichment_hbase_acl_configured_flag_file = status_params.enrichment_hbase_acl_configured_flag_file
enrichment_maxmind_configured_flag_file = status_params.enrichment_maxmind_configured_flag_file
indexing_configured_flag_file = status_params.indexing_configured_flag_file
@@ -146,6 +150,9 @@ solr_user = config['configurations']['solr-config-env']['solr_config_user']
solr_principal_name = config['configurations']['solr-config-env']['solr_principal_name']
solr_keytab_path = config['configurations']['solr-config-env']['solr_keytab_path']
+# HDFS
+hdfs_url = status_params.hdfs_url
+
# Storm
storm_rest_addr = status_params.storm_rest_addr
@@ -164,8 +171,6 @@ if has_kafka_host:
kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
kafka_brokers += ':' + kafka_broker_port
-metron_apps_hdfs_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
-
# the double "format" is not an error - we are pulling in a jinja-templated param. This is a bit of a hack, but works
# well enough until we find a better way via Ambari
metron_temp_grok_path = format(format(config['configurations']['metron-rest-env']['metron_temp_grok_path']))
@@ -208,6 +213,11 @@ HdfsResource = functools.partial(
enrichment_hbase_provider_impl = 'org.apache.metron.hbase.HTableProvider'
enrichment_hbase_table = status_params.enrichment_hbase_table
enrichment_hbase_cf = status_params.enrichment_hbase_cf
+# coprocessor config for enrichment list
+enrichment_list_hbase_provider_impl = status_params.enrichment_list_hbase_provider_impl
+enrichment_list_hbase_coprocessor_impl = status_params.enrichment_list_hbase_coprocessor_impl
+enrichment_list_hbase_table = status_params.enrichment_list_hbase_table
+enrichment_list_hbase_cf = status_params.enrichment_list_hbase_cf
update_hbase_table = status_params.update_hbase_table
update_hbase_cf = status_params.update_hbase_cf
@@ -475,4 +485,4 @@ knox_home = os.path.join(stack_root, "current", "knox-server")
knox_hosts = default("/clusterHostInfo/knox_gateway_hosts", [])
knox_host = ''
if not len(knox_hosts) == 0:
- knox_host = knox_hosts[0]
\ No newline at end of file
+ knox_host = knox_hosts[0]
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index a9fc8f8..fc5daac 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -44,11 +44,16 @@ enrichment_input_topic = config['configurations']['metron-enrichment-env']['enri
enrichment_kafka_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_kafka_configured'
enrichment_kafka_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_kafka_acl_configured'
enrichment_hbase_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_configured'
+enrichment_hbase_coprocessor_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_coprocessor_configured'
enrichment_hbase_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_acl_configured'
enrichment_maxmind_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_maxmind_configured'
enrichment_hbase_table = config['configurations']['metron-enrichment-env']['enrichment_hbase_table']
enrichment_hbase_cf = config['configurations']['metron-enrichment-env']['enrichment_hbase_cf']
+enrichment_list_hbase_provider_impl = config['configurations']['metron-enrichment-env']['enrichment_list_hbase_provider_impl']
+enrichment_list_hbase_coprocessor_impl = config['configurations']['metron-enrichment-env']['enrichment_list_hbase_coprocessor_impl']
+enrichment_list_hbase_table = config['configurations']['metron-enrichment-env']['enrichment_list_hbase_table']
+enrichment_list_hbase_cf = config['configurations']['metron-enrichment-env']['enrichment_list_hbase_cf']
threatintel_hbase_table = config['configurations']['metron-enrichment-env']['threatintel_hbase_table']
threatintel_hbase_cf = config['configurations']['metron-enrichment-env']['threatintel_hbase_cf']
update_hbase_table = config['configurations']['metron-indexing-env']['update_hbase_table']
@@ -99,6 +104,12 @@ metron_alerts_ui_port = config['configurations']['metron-alerts-ui-env']['metron
metron_management_ui_host = default("/clusterHostInfo/metron_management_ui_hosts", [hostname])[0]
metron_management_ui_port = config['configurations']['metron-management-ui-env']['metron_management_ui_port']
+# HDFS
+# This should always grab the full namenode url, e.g. "hdfs://mynamenodehost:8020"
+# In the case of namenode HA, it will refer to the nameservice ID
+# https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithNFS.html
+hdfs_url = config["configurations"]["core-site"]["fs.defaultFS"]
+
# Storm
storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr']
@@ -130,4 +141,4 @@ pcap_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_
metron_user_hdfs_dir_configured_flag_file = metron_zookeeper_config_path + '/../metron_user_hdfs_dir_configured'
# Knox
-metron_knox_installed_flag_file = metron_zookeeper_config_path + '/../metron_knox_installed'
\ No newline at end of file
+metron_knox_installed_flag_file = metron_zookeeper_config_path + '/../metron_knox_installed'
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 1c40085..fbffe28 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -63,6 +63,7 @@ Source16: metron-profiler-spark-%{full_version}-archive.tar.gz
Source17: metron-profiler-repl-%{full_version}-archive.tar.gz
Source18: metron-parsing-storm-%{full_version}-archive.tar.gz
Source19: metron-parsers-%{full_version}-archive.tar.gz
+Source20: metron-hbase-server-%{full_version}-archive.tar.gz
%description
Apache Metron provides a scalable advanced security analytics framework
@@ -105,6 +106,7 @@ tar -xzf %{SOURCE16} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE17} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE18} -C %{buildroot}%{metron_home}
tar -xzf %{SOURCE19} -C %{buildroot}%{metron_home}
+tar -xzf %{SOURCE20} -C %{buildroot}%{metron_home}
install %{buildroot}%{metron_home}/bin/metron-management-ui %{buildroot}/etc/init.d/
install %{buildroot}%{metron_home}/bin/metron-alerts-ui %{buildroot}/etc/init.d/
@@ -654,6 +656,25 @@ This package installs the Metron Profiler for the Stellar REPL %{metron_home}
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+%package hbase-server
+Summary: Metron HBase Server Files
+Group: Applications/Internet
+Provides: hbase-server = %{version}
+
+%description hbase-server
+This package installs the Metron HBase Server files
+
+%files hbase-server
+%defattr(-,root,root,755)
+%dir %{metron_root}
+%dir %{metron_home}
+%dir %{metron_home}/coprocessor
+%dir %{metron_home}/bin
+%{metron_home}/bin/load_enrichment_coprocessor.sh
+%attr(0644,root,root) %{metron_home}/coprocessor/metron-hbase-server-%{full_version}-uber.jar
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
%post config
chkconfig --add metron-management-ui
chkconfig --add metron-alerts-ui
@@ -663,6 +684,8 @@ chkconfig --del metron-management-ui
chkconfig --del metron-alerts-ui
%changelog
+* Mon Apr 8 2019 Apache Metron <de...@metron.apache.og> - 0.7.1
+- Updat metron SPEC to include metron-hbase-server for enrichment coprocessor
* Tue Mar 12 2019 Apache Metron <de...@metron.apache.og> - 0.7.1
- Split metron-enrichment into submodules
* Thu Dec 27 2018 Apache Metron <de...@metron.apache.og> - 0.7.1
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index d8d117a..1d580b0 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -138,6 +138,12 @@
</includes>
</resource>
<resource>
+ <directory>${metron_dir}/metron-platform/metron-hbase-server/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
+ <resource>
<directory>${metron_dir}/metron-platform/metron-indexing/target/</directory>
<includes>
<include>*.tar.gz</include>
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index 3201f21..c76a402 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -733,7 +733,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available
### `GET /api/v1/sensor/enrichment/config/list/available/enrichments`
* Description: Lists the available enrichments
* Returns:
- * 200 - Returns a list of available enrichments
+ * 200 - Returns a list of available enrichments in lexicographical order
### `GET /api/v1/sensor/enrichment/config/list/available/threat/triage/aggregators`
* Description: Lists the available threat triage aggregators
diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml
index cc58bf8..a01267b 100644
--- a/metron-interface/metron-rest/pom.xml
+++ b/metron-interface/metron-rest/pom.xml
@@ -42,6 +42,15 @@
<jwt.version>4.41.2</jwt.version>
</properties>
<dependencies>
+ <dependency>
+ <!-- There's an issue with the Javassist versions used by our version of Powermock. Keep the explicit test dep at the top
+ so it takes precedence during testing - https://github.com/powermock/powermock/issues/729
+ -->
+ <groupId>org.javassist</groupId>
+ <artifactId>javassist</artifactId>
+ <version>3.20.0-GA</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
@@ -350,6 +359,12 @@
<artifactId>powermock-module-junit4</artifactId>
<version>${powermock.version}</version>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>javassist</artifactId>
+ <groupId>org.javassist</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
@@ -387,6 +402,10 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>javassist</artifactId>
+ <groupId>org.javassist</groupId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HBaseConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HBaseConfig.java
index c96b9e8..7ce16f9 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HBaseConfig.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HBaseConfig.java
@@ -17,17 +17,23 @@
*/
package org.apache.metron.rest.config;
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
import org.apache.metron.hbase.HTableProvider;
-import org.apache.metron.hbase.client.UserSettingsClient;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.hbase.client.HBaseClient;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.service.GlobalConfigService;
+import org.apache.metron.rest.user.UserSettingsClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
-
@Configuration
@Profile("!" + TEST_PROFILE)
public class HBaseConfig {
@@ -52,4 +58,25 @@ public class HBaseConfig {
}, new HTableProvider());
return userSettingsClient;
}
+
+ @Bean()
+ public HBaseClient hBaseClient() {
+ Map<String, Object> restConfig = null;
+ try {
+ restConfig = globalConfigService.get();
+ } catch (RestException e) {
+ throw new IllegalStateException("Unable to retrieve the global config.", e);
+ }
+ TableProvider provider = null;
+ try {
+ provider = TableProvider
+ .create((String) restConfig.get(EnrichmentConfigurations.TABLE_PROVIDER),
+ HTableProvider::new);
+ } catch (ClassNotFoundException | InstantiationException | InvocationTargetException | IllegalAccessException | NoSuchMethodException e) {
+ throw new IllegalStateException("Unable to create table provider", e);
+ }
+ return new HBaseClient(provider, HBaseConfiguration.create(),
+ (String) restConfig.get(EnrichmentConfigurations.TABLE_NAME));
+ }
+
}
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorEnrichmentConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorEnrichmentConfigService.java
index 817a57d..18f9273 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorEnrichmentConfigService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorEnrichmentConfigService.java
@@ -17,12 +17,10 @@
*/
package org.apache.metron.rest.service;
-import org.apache.metron.common.aggregator.Aggregators;
-import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
-import org.apache.metron.rest.RestException;
-
import java.util.List;
import java.util.Map;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.rest.RestException;
public interface SensorEnrichmentConfigService {
@@ -36,7 +34,7 @@ public interface SensorEnrichmentConfigService {
boolean delete(String name) throws RestException;
- List<String> getAvailableEnrichments();
+ List<String> getAvailableEnrichments() throws RestException;
List<String> getAvailableThreatTriageAggregators();
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
index bf035e2..6092ce2 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
@@ -17,34 +17,33 @@
*/
package org.apache.metron.rest.service.impl;
+import static org.apache.metron.rest.MetronRestConstants.KAFKA_TOPICS_ESCALATION_PROPERTY;
+import static org.apache.metron.rest.MetronRestConstants.METRON_ESCALATION_TIMESTAMP_FIELD;
+import static org.apache.metron.rest.MetronRestConstants.METRON_ESCALATION_USER_FIELD;
+
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
import org.apache.metron.common.system.Clock;
import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.hbase.client.UserSettingsClient;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.AlertsUIUserSettings;
import org.apache.metron.rest.security.SecurityUtils;
import org.apache.metron.rest.service.AlertsUIService;
import org.apache.metron.rest.service.KafkaService;
+import org.apache.metron.rest.user.UserSettingsClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import static org.apache.metron.rest.MetronRestConstants.KAFKA_TOPICS_ESCALATION_PROPERTY;
-import static org.apache.metron.rest.MetronRestConstants.METRON_ESCALATION_TIMESTAMP_FIELD;
-import static org.apache.metron.rest.MetronRestConstants.METRON_ESCALATION_USER_FIELD;
-
/**
* The default service layer implementation of {@link AlertsUIService}.
*
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
index 293b113..5c0f2e0 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java
@@ -18,6 +18,13 @@
package org.apache.metron.rest.service.impl;
import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.curator.framework.CuratorFramework;
import org.apache.metron.common.aggregator.Aggregators;
import org.apache.metron.common.configuration.ConfigurationType;
@@ -25,20 +32,13 @@ import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.common.configuration.EnrichmentConfigurations;
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
+import org.apache.metron.hbase.client.HBaseClient;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.service.SensorEnrichmentConfigService;
-import org.apache.metron.common.zookeeper.ZKConfigurationsCache;
import org.apache.zookeeper.KeeperException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
@Service
public class SensorEnrichmentConfigServiceImpl implements SensorEnrichmentConfigService {
@@ -48,11 +48,16 @@ public class SensorEnrichmentConfigServiceImpl implements SensorEnrichmentConfig
private ConfigurationsCache cache;
+ private HBaseClient hBaseClient;
+
@Autowired
- public SensorEnrichmentConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client, ConfigurationsCache cache) {
+ public SensorEnrichmentConfigServiceImpl(final ObjectMapper objectMapper,
+ final CuratorFramework client, final ConfigurationsCache cache,
+ final HBaseClient hBaseClient) {
this.objectMapper = objectMapper;
this.client = client;
this.cache = cache;
+ this.hBaseClient = hBaseClient;
}
@Override
@@ -102,13 +107,18 @@ public class SensorEnrichmentConfigServiceImpl implements SensorEnrichmentConfig
return true;
}
+ /**
+ * Lexicographically sorted list of available enrichments.
+ */
@Override
- public List<String> getAvailableEnrichments() {
- return new ArrayList<String>() {{
- add("geo");
- add("host");
- add("whois");
- }};
+ public List<String> getAvailableEnrichments() throws RestException {
+ try {
+ List<String> enrichments = hBaseClient.readRecords();
+ enrichments.sort(Comparator.naturalOrder());
+ return enrichments;
+ } catch (IOException e) {
+ throw new RestException("Unable to retrieve enrichments", e);
+ }
}
@Override
diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/client/UserSettingsClient.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/user/UserSettingsClient.java
similarity index 99%
rename from metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/client/UserSettingsClient.java
rename to metron-interface/metron-rest/src/main/java/org/apache/metron/rest/user/UserSettingsClient.java
index f492497..a08b775 100644
--- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/client/UserSettingsClient.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/user/UserSettingsClient.java
@@ -15,8 +15,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.metron.hbase.client;
+package org.apache.metron.rest.user;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Optional;
+import java.util.function.Supplier;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
@@ -30,15 +38,6 @@ import org.apache.metron.hbase.TableProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Optional;
-import java.util.function.Supplier;
-
public class UserSettingsClient {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HBaseConfigTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HBaseConfigTest.java
index a527e6d..0d45f18 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HBaseConfigTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HBaseConfigTest.java
@@ -17,8 +17,8 @@
*/
package org.apache.metron.rest.config;
-import static org.apache.metron.hbase.client.UserSettingsClient.USER_SETTINGS_HBASE_CF;
-import static org.apache.metron.hbase.client.UserSettingsClient.USER_SETTINGS_HBASE_TABLE;
+import static org.apache.metron.rest.user.UserSettingsClient.USER_SETTINGS_HBASE_CF;
+import static org.apache.metron.rest.user.UserSettingsClient.USER_SETTINGS_HBASE_TABLE;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
@@ -29,8 +29,11 @@ import static org.powermock.api.mockito.PowerMockito.whenNew;
import java.util.HashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
import org.apache.metron.hbase.HTableProvider;
+import org.apache.metron.hbase.mock.MockHBaseTableProvider;
import org.apache.metron.rest.service.GlobalConfigService;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -67,5 +70,21 @@ public class HBaseConfigTest {
verifyZeroInteractions(htableProvider);
}
+ @Test
+ public void hBaseClientShouldBeCreatedWithDefaultProvider() throws Exception {
+ when(globalConfigService.get()).thenReturn(new HashMap<String, Object>() {{
+ put(EnrichmentConfigurations.TABLE_NAME, "enrichment_list_hbase_table_name");
+ }});
+ Assert.assertNotNull(hBaseConfig.hBaseClient());
+ }
+
+ @Test
+ public void hBaseClientShouldBeCreatedWithSpecifiedProvider() throws Exception {
+ when(globalConfigService.get()).thenReturn(new HashMap<String, Object>() {{
+ put(EnrichmentConfigurations.TABLE_PROVIDER, MockHBaseTableProvider.class.getName());
+ put(EnrichmentConfigurations.TABLE_NAME, "enrichment_list_hbase_table_name");
+ }});
+ Assert.assertNotNull(hBaseConfig.hBaseClient());
+ }
}
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
index 942ff78..920c7ab 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
@@ -23,7 +23,10 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
import kafka.admin.AdminUtils$;
@@ -35,12 +38,15 @@ import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.common.zookeeper.ZKConfigurationsCache;
-import org.apache.metron.hbase.client.UserSettingsClient;
+import org.apache.metron.hbase.client.HBaseClient;
import org.apache.metron.hbase.mock.MockHBaseTableProvider;
import org.apache.metron.integration.ComponentRunner;
import org.apache.metron.integration.UnableToStartException;
@@ -56,6 +62,7 @@ import org.apache.metron.rest.mock.MockStormCLIClientWrapper;
import org.apache.metron.rest.mock.MockStormRestTemplate;
import org.apache.metron.rest.service.impl.PcapToPdmlScriptWrapper;
import org.apache.metron.rest.service.impl.StormCLIWrapper;
+import org.apache.metron.rest.user.UserSettingsClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
@@ -185,12 +192,30 @@ public class TestConfig {
return AdminUtils$.MODULE$;
}
-
@Bean()
public UserSettingsClient userSettingsClient() throws RestException, IOException {
return new UserSettingsClient(new MockHBaseTableProvider().addToCache("user_settings", "cf"), Bytes.toBytes("cf"));
}
+ @Bean()
+ public HBaseClient hBaseClient() throws RestException, IOException {
+ final String cf = "t";
+ final String cq = "v";
+ HTableInterface table = MockHBaseTableProvider.addToCache("enrichment_list", cf);
+ List<String> enrichmentTypes = new ArrayList<String>() {{
+ add("foo");
+ add("bar");
+ add("baz");
+ }};
+ for (String type : enrichmentTypes) {
+ Put put = new Put(Bytes.toBytes(type));
+ put.addColumn(Bytes.toBytes(cf), Bytes.toBytes(cq), "{}".getBytes(StandardCharsets.UTF_8));
+ table.put(put);
+ }
+ return new HBaseClient(new MockHBaseTableProvider(), HBaseConfiguration.create(),
+ "enrichment_list");
+ }
+
@Bean
public JobManager jobManager() {
return new InMemoryJobManager();
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
index 15a2370..d77e3ae 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java
@@ -17,8 +17,21 @@
*/
package org.apache.metron.rest.controller;
+import static org.apache.metron.integration.utils.TestUtils.assertEventually;
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.rest.service.SensorEnrichmentConfigService;
+import org.hamcrest.core.IsCollectionContaining;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -31,18 +44,6 @@ import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.WebApplicationContext;
-import static org.apache.metron.integration.utils.TestUtils.assertEventually;
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
-import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
-
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
@ActiveProfiles(TEST_PROFILE)
@@ -141,6 +142,9 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
.andExpect(status().isUnauthorized());
}
+ /**
+ * Note: <a href="https://github.com/json-path/JsonPath#path-examples">JSON Path examples</a>
+ */
@Test
public void test() throws Exception {
sensorEnrichmentConfigService.delete("broTest");
@@ -241,9 +245,8 @@ public class SensorEnrichmentConfigControllerIntegrationTest {
this.mockMvc.perform(get(sensorEnrichmentConfigUrl + "/list/available/enrichments").with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$[0]").value("geo"))
- .andExpect(jsonPath("$[1]").value("host"))
- .andExpect(jsonPath("$[2]").value("whois"));
+ .andExpect(jsonPath("$.length()").value("3"))
+ .andExpect(jsonPath("$.*").value(IsCollectionContaining.hasItems("foo", "bar", "baz")));
this.mockMvc.perform(get(sensorEnrichmentConfigUrl + "/list/available/threat/triage/aggregators").with(httpBasic(user,password)))
.andExpect(status().isOk())
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/AlertsUIServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/AlertsUIServiceImplTest.java
index 545d7f9..016a08b 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/AlertsUIServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/AlertsUIServiceImplTest.java
@@ -20,7 +20,6 @@ package org.apache.metron.rest.service.impl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -29,24 +28,20 @@ import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
import static org.powermock.api.mockito.PowerMockito.mock;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.Optional;
-
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.system.FakeClock;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.model.AlertsUIUserSettings;
-import org.apache.metron.hbase.client.UserSettingsClient;
-import org.apache.metron.rest.service.AlertsUIService;
import org.apache.metron.rest.service.KafkaService;
+import org.apache.metron.rest.user.UserSettingsClient;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java
index 0a78f4a..3d07da5 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java
@@ -17,13 +17,24 @@
*/
package org.apache.metron.rest.service.impl;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.DeleteBuilder;
-import org.apache.curator.framework.api.GetChildrenBuilder;
-import org.apache.curator.framework.api.GetDataBuilder;
import org.apache.curator.framework.api.SetDataBuilder;
import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.common.configuration.EnrichmentConfigurations;
@@ -31,6 +42,7 @@ import org.apache.metron.common.configuration.enrichment.EnrichmentConfig;
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.common.configuration.enrichment.threatintel.ThreatIntelConfig;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
+import org.apache.metron.hbase.client.HBaseClient;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.service.SensorEnrichmentConfigService;
import org.apache.zookeeper.KeeperException;
@@ -40,20 +52,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
@SuppressWarnings("ALL")
public class SensorEnrichmentConfigServiceImplTest {
@Rule
@@ -84,13 +82,15 @@ public class SensorEnrichmentConfigServiceImplTest {
public static String broJson;
ConfigurationsCache cache;
+ private HBaseClient hBaseClient;
@Before
public void setUp() throws Exception {
objectMapper = mock(ObjectMapper.class);
curatorFramework = mock(CuratorFramework.class);
cache = mock(ConfigurationsCache.class);
- sensorEnrichmentConfigService = new SensorEnrichmentConfigServiceImpl(objectMapper, curatorFramework, cache);
+ hBaseClient = mock(HBaseClient.class);
+ sensorEnrichmentConfigService = new SensorEnrichmentConfigServiceImpl(objectMapper, curatorFramework, cache, hBaseClient);
}
@@ -211,8 +211,15 @@ public class SensorEnrichmentConfigServiceImplTest {
}
@Test
- public void getAvailableEnrichmentsShouldReturnEnrichments() throws Exception {
+ public void getAvailableEnrichmentsShouldReturnEnrichmentsSorted() throws Exception {
+ when(hBaseClient.readRecords()).thenReturn(new ArrayList<String>() {{
+ add("geo");
+ add("whois");
+ add("host");
+ add("a-new-one");
+ }});
assertEquals(new ArrayList<String>() {{
+ add("a-new-one");
add("geo");
add("host");
add("whois");
diff --git a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/UserSettingsClientTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/user/UserSettingsClientTest.java
similarity index 93%
rename from metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/UserSettingsClientTest.java
rename to metron-interface/metron-rest/src/test/java/org/apache/metron/rest/user/UserSettingsClientTest.java
index 55a28ad..5b4f786 100644
--- a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/UserSettingsClientTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/user/UserSettingsClientTest.java
@@ -15,36 +15,34 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.metron.hbase.client;
+package org.apache.metron.rest.user;
+import static org.apache.metron.rest.user.UserSettingsClient.USER_SETTINGS_HBASE_CF;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Supplier;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.function.Supplier;
-
-import static org.apache.metron.hbase.client.UserSettingsClient.USER_SETTINGS_HBASE_CF;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
public class UserSettingsClientTest {
@Rule
@@ -75,7 +73,7 @@ public class UserSettingsClientTest {
when(userSettingsTable.get(get)).thenReturn(result);
UserSettingsClient userSettingsClient = new UserSettingsClient(userSettingsTable, cf);
- Assert.assertEquals("userSettings1String", userSettingsClient.findOne("user1", "type").get());
+ assertEquals("userSettings1String", userSettingsClient.findOne("user1", "type").get());
assertFalse(userSettingsClient.findOne("missingUser", "type").isPresent());
}
diff --git a/metron-interface/metron-rest/src/test/resources/zookeeper/global.json b/metron-interface/metron-rest/src/test/resources/zookeeper/global.json
index 396896f..ad344e1 100644
--- a/metron-interface/metron-rest/src/test/resources/zookeeper/global.json
+++ b/metron-interface/metron-rest/src/test/resources/zookeeper/global.json
@@ -1,4 +1,4 @@
{
"update.hbase.table" : "updates",
"update.hbase.cf" : "t"
-}
\ No newline at end of file
+}
diff --git a/metron-platform/metron-common/README.md b/metron-platform/metron-common/README.md
index 40f1289..5144be7 100644
--- a/metron-platform/metron-common/README.md
+++ b/metron-platform/metron-common/README.md
@@ -80,41 +80,44 @@ This configuration is stored in zookeeper, but looks something like
Various parts of our stack uses the global config are documented throughout the Metron documentation,
but a convenient index is provided here:
-| Property Name | Subsystem | Type | Ambari Property |
-|---------------------------------------------------------------------------------------------------------------------|---------------|------------|----------------------------------------|
-| [`es.clustername`](../metron-elasticsearch#esclustername) | Indexing | String | `es_cluster_name` |
-| [`es.ip`](../metron-elasticsearch#esip) | Indexing | String | `es_hosts` & `es_port` |
-| [`es.port`](../metron-elasticsearch#esport) | Indexing | String | N/A |
-| [`es.date.format`](../metron-elasticsearch#esdateformat) | Indexing | String | `es_date_format` |
-| [`es.client.settings`](../metron-elasticsearch#esclientsettings) | Indexing | Object | N/A |
-| [`solr.zookeeper`](../metron-solr#configuration) | Indexing | String | `solr_zookeeper_url` |
-| [`solr.commitPerBatch`](../metron-solr#configuration) | Indexing | String | N/A |
-| [`solr.commit.soft`](../metron-solr#configuration) | Indexing | String | N/A |
-| [`solr.commit.waitSearcher`](../metron-solr#configuration) | Indexing | String | N/A |
-| [`solr.commit.waitFlush`](../metron-solr#configuration) | Indexing | String | N/A |
-| [`solr.collection`](../metron-solr#configuration) | Indexing | String | N/A |
-| [`solr.http.config`](../metron-solr#configuration) | Indexing | String | N/A |
-| [`fieldValidations`](#validation-framework) | Parsing | Object | N/A |
-| [`parser.error.topic`](../metron-parsers#parsererrortopic) | Parsing | String | `parser_error_topic` |
-| [`stellar.function.paths`](../../metron-stellar/stellar-common#stellarfunctionpaths) | Stellar | CSV String | N/A |
-| [`stellar.function.resolver.includes`](../../metron-stellar/stellar-common#stellarfunctionresolverincludesexcludes) | Stellar | CSV String | N/A |
-| [`stellar.function.resolver.excludes`](../../metron-stellar/stellar-common#stellarfunctionresolverincludesexcludes) | Stellar | CSV String | N/A |
-| [`profiler.period.duration`](../../metron-analytics/metron-profiler-storm#profilerperiodduration) | Profiler | Integer | `profiler_period_duration` |
-| [`profiler.period.duration.units`](../../metron-analytics/metron-profiler-storm#profilerperioddurationunits) | Profiler | String | `profiler_period_units` |
-| [`profiler.client.period.duration`](../../metron-analytics/metron-profiler-storm#profilerperiodduration) | Profiler | Integer | `profiler_period_duration` |
-| [`profiler.client.period.duration.units`](../../metron-analytics/metron-profiler-storm#profilerperioddurationunits) | Profiler | String | `profiler_period_units` |
-| [`profiler.writer.batchSize`](../../metron-analytics/metron-profiler-storm/#profilerwriterbatchsize) | Profiler | Integer | `profiler_kafka_writer_batch_size` |
-| [`profiler.writer.batchTimeout`](../../metron-analytics/metron-profiler-storm/#profilerwriterbatchtimeout) | Profiler | Integer | `profiler_kafka_writer_batch_timeout` |
-| [`update.hbase.table`](../metron-indexing#updatehbasetable) | REST/Indexing | String | `update_hbase_table` |
-| [`update.hbase.cf`](../metron-indexing#updatehbasecf) | REST/Indexing | String | `update_hbase_cf` |
-| [`user.settings.hbase.table`](../metron-interface/metron-rest) | REST/Indexing | String | `user_settings_hbase_table` |
-| [`user.settings.hbase.cf`](../metron-interface/metron-rest) | REST/Indexing | String | `user_settings_hbase_cf` |
-| [`geo.hdfs.file`](../metron-enrichment#geohdfsfile) | Enrichment | String | `geo_hdfs_file` |
-| [`enrichment.writer.batchSize`](../metron-enrichment#enrichmentwriterbatchsize) | Enrichment | Integer | `enrichment_kafka_writer_batch_size` |
-| [`enrichment.writer.batchTimeout`](../metron-enrichment#enrichmentwriterbatchtimeout) | Enrichment | Integer | `enrichment_kafka_writer_batch_timeout`|
-| [`geo.hdfs.file`](../metron-enrichment#geohdfsfile) | Enrichment | String | `geo_hdfs_file` |
-| [`source.type.field`](../../metron-interface/metron-alerts#sourcetypefield) | UI | String | `source_type_field` |
-| [`threat.triage.score.field`](../../metron-interface/metron-alerts#threattriagescorefield) | UI | String | `threat_triage_score_field` |
+| Property Name | Subsystem | Type | Ambari Property |
+|-----------------------------------------------------------------------------------------------------------------------|---------------|------------|-----------------------------------------|
+| [`es.clustername`](../metron-elasticsearch#esclustername) | Indexing | String | `es_cluster_name` |
+| [`es.ip`](../metron-elasticsearch#esip) | Indexing | String | `es_hosts` & `es_port` |
+| [`es.port`](../metron-elasticsearch#esport) | Indexing | String | N/A |
+| [`es.date.format`](../metron-elasticsearch#esdateformat) | Indexing | String | `es_date_format` |
+| [`es.client.settings`](../metron-elasticsearch#esclientsettings) | Indexing | Object | N/A |
+| [`solr.zookeeper`](../metron-solr#configuration) | Indexing | String | `solr_zookeeper_url` |
+| [`solr.commitPerBatch`](../metron-solr#configuration) | Indexing | String | N/A |
+| [`solr.commit.soft`](../metron-solr#configuration) | Indexing | String | N/A |
+| [`solr.commit.waitSearcher`](../metron-solr#configuration) | Indexing | String | N/A |
+| [`solr.commit.waitFlush`](../metron-solr#configuration) | Indexing | String | N/A |
+| [`solr.collection`](../metron-solr#configuration) | Indexing | String | N/A |
+| [`solr.http.config`](../metron-solr#configuration) | Indexing | String | N/A |
+| [`fieldValidations`](#validation-framework) | Parsing | Object | N/A |
+| [`parser.error.topic`](../metron-parsers#parsererrortopic) | Parsing | String | `parser_error_topic` |
+| [`stellar.function.paths`](../../metron-stellar/stellar-common#stellarfunctionpaths) | Stellar | CSV String | N/A |
+| [`stellar.function.resolver.includes`](../../metron-stellar/stellar-common#stellarfunctionresolverincludesexcludes) | Stellar | CSV String | N/A |
+| [`stellar.function.resolver.excludes`](../../metron-stellar/stellar-common#stellarfunctionresolverincludesexcludes) | Stellar | CSV String | N/A |
+| [`profiler.period.duration`](../../metron-analytics/metron-profiler-storm#profilerperiodduration) | Profiler | Integer | `profiler_period_duration` |
+| [`profiler.period.duration.units`](../../metron-analytics/metron-profiler-storm#profilerperioddurationunits) | Profiler | String | `profiler_period_units` |
+| [`profiler.client.period.duration`](../../metron-analytics/metron-profiler-storm#profilerperiodduration) | Profiler | Integer | `profiler_period_duration` |
+| [`profiler.client.period.duration.units`](../../metron-analytics/metron-profiler-storm#profilerperioddurationunits) | Profiler | String | `profiler_period_units` |
+| [`profiler.writer.batchSize`](../../metron-analytics/metron-profiler-storm/#profilerwriterbatchsize) | Profiler | Integer | `profiler_kafka_writer_batch_size` |
+| [`profiler.writer.batchTimeout`](../../metron-analytics/metron-profiler-storm/#profilerwriterbatchtimeout) | Profiler | Integer | `profiler_kafka_writer_batch_timeout` |
+| [`update.hbase.table`](../metron-indexing#updatehbasetable) | REST/Indexing | String | `update_hbase_table` |
+| [`update.hbase.cf`](../metron-indexing#updatehbasecf) | REST/Indexing | String | `update_hbase_cf` |
+| [`user.settings.hbase.table`](../metron-interface/metron-rest) | REST/Indexing | String | `user_settings_hbase_table` |
+| [`user.settings.hbase.cf`](../metron-interface/metron-rest) | REST/Indexing | String | `user_settings_hbase_cf` |
+| [`geo.hdfs.file`](../metron-enrichment/metron-enrichment-common#geohdfsfile) | Enrichment | String | `geo_hdfs_file` |
+| [`enrichment.writer.batchSize`](../metron-enrichment/metron-enrichment-common#enrichmentwriterbatchsize) | Enrichment | Integer | `enrichment_kafka_writer_batch_size` |
+| [`enrichment.writer.batchTimeout`](../metron-enrichment/metron-enrichment-common#enrichmentwriterbatchtimeout) | Enrichment | Integer | `enrichment_kafka_writer_batch_timeout` |
+| [`enrichment.list.hbase.provider.impl`](../metron-hbase-server#enrichmentlisthbaseproviderimpl) | Enrichment | String | `enrichment_list_hbase_provider_impl` |
+| [`enrichment.list.hbase.table`](../metron-hbase-server#enrichmentlisthbasetable) | Enrichment | String | `enrichment_list_hbase_table` |
+| [`enrichment.list.hbase.cf`](../metron-hbase-server#enrichmentlisthbasecf) | Enrichment | String | `enrichment_list_hbase_cf` |
+| [`geo.hdfs.file`](../metron-enrichment#geohdfsfile) | Enrichment | String | `geo_hdfs_file` |
+| [`source.type.field`](../../metron-interface/metron-alerts#sourcetypefield) | UI | String | `source_type_field` |
+| [`threat.triage.score.field`](../../metron-interface/metron-alerts#threattriagescorefield) | UI | String | `threat_triage_score_field` |
## Note Configs in Ambari
If a field is managed via ambari, you should change the field via
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java
index 9136725..5325b5f 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java
@@ -31,9 +31,14 @@ import java.util.List;
* global config and provided here for convenience.
*/
public class EnrichmentConfigurations extends Configurations {
+ // Writer batch params
public static final Integer DEFAULT_KAFKA_BATCH_SIZE = 15;
public static final String BATCH_SIZE_CONF = "enrichment.writer.batchSize";
public static final String BATCH_TIMEOUT_CONF = "enrichment.writer.batchTimeout";
+ // Enrichment list table params - assumes HBase implementation
+ public static final String TABLE_PROVIDER = "enrichment.list.hbase.provider.impl";
+ public static final String TABLE_NAME = "enrichment.list.hbase.table";
+ public static final String COLUMN_FAMILY = "enrichment.list.hbase.cf";
public SensorEnrichmentConfig getSensorEnrichmentConfig(String sensorType) {
return (SensorEnrichmentConfig) getConfigurations().get(getKey(sensorType));
diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml
index c4b2bf6..fe59f8c 100644
--- a/metron-platform/metron-data-management/pom.xml
+++ b/metron-platform/metron-data-management/pom.xml
@@ -346,6 +346,18 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${global_jar_version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${global_shade_version}</version>
<executions>
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/HBaseUtil.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/HBaseUtil.java
index c9c6424..48a9ba3 100644
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/HBaseUtil.java
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/HBaseUtil.java
@@ -18,6 +18,16 @@
package org.apache.metron.dataloads.hbase.mr;
import com.google.common.base.Joiner;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -26,26 +36,36 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import java.io.*;
-import java.util.AbstractMap;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
+/**
+ * Spin up a test HBase cluster
+ */
public enum HBaseUtil {
- INSTANCE;
- public Map.Entry<HBaseTestingUtility,Configuration> create(boolean startMRCluster) throws Exception {
- Configuration config = HBaseConfiguration.create();
- config.set("hbase.master.hostname", "localhost");
- config.set("hbase.regionserver.hostname", "localhost");
- HBaseTestingUtility testUtil = new HBaseTestingUtility(config);
+ INSTANCE;
- testUtil.startMiniCluster(1);
- if(startMRCluster) {
- testUtil.startMiniMapReduceCluster();
- }
- return new AbstractMap.SimpleEntry<>(testUtil, config);
+ public Map.Entry<HBaseTestingUtility, Configuration> create(boolean startMRCluster)
+ throws Exception {
+ return create(startMRCluster, null);
+ }
+
+ public Map.Entry<HBaseTestingUtility, Configuration> create(boolean startMRCluster,
+ Configuration extraConfig) throws Exception {
+ Configuration config = HBaseConfiguration.create();
+ config.set("hbase.master.hostname", "localhost");
+ config.set("hbase.regionserver.hostname", "localhost");
+ if (null != extraConfig) {
+ for (Entry<String, String> entry : extraConfig) {
+ config.set(entry.getKey(), entry.getValue());
+ }
+ }
+ HBaseTestingUtility testUtil = new HBaseTestingUtility(config);
+
+ testUtil.startMiniCluster(1);
+ if (startMRCluster) {
+ testUtil.startMiniMapReduceCluster();
}
+ return new AbstractMap.SimpleEntry<>(testUtil, config);
+ }
+
public void writeFile(String contents, Path filename, FileSystem fs) throws IOException {
FSDataOutputStream os = fs.create(filename, true);
PrintWriter pw = new PrintWriter(new OutputStreamWriter(os));
diff --git a/metron-platform/metron-enrichment/README.md b/metron-platform/metron-enrichment/README.md
index 1e2d055..4246528 100644
--- a/metron-platform/metron-enrichment/README.md
+++ b/metron-platform/metron-enrichment/README.md
@@ -38,3 +38,17 @@ There is currently one option for running enrichments in Metron, which is as a S
* metron-enrichment-common - this module houses the prepackaged enrichment configuration by sensor. It also contains the core enrichment and threat intelligence processing functionality.
* metron-common-storm - this module is home to Storm-specific code such as Flux files and Storm Bolts.
+## Enrichments List
+
+Metron provides an HBase table for storing enrichments. The rowkeys are a combination of a salt (for managing
+RegionServer hotspotting), indicator (this would be the search value, e.g. "192.168.1.1"), and type (whois, geoip, etc.).
+
+This approach performs well for both inserts and lookups, but poses a challenge when looking to get an
+up-to-date list of all the current enrichments. This is of particular concern for the Management UI where
+it's desirable to provide an accurate list of all available enrichment types.
+A table scan is undesirable because it results in a performance hit for inserts and reads.
+The alternative approach that mitigates these performance bottlenecks is to leverage a custom HBase
+Coprocessor which will listen to postPut calls from the RegionServer, extract the enrichment type from the rowkey, and
+perform an insert into a separate `enrichment_list` HBase table.
+
+See more about configuring the coprocessor here [Enrichment Coprocessor](../metron-hbase-server/#enrichment-coprocessor)
diff --git a/metron-platform/metron-hbase-server/README.md b/metron-platform/metron-hbase-server/README.md
new file mode 100644
index 0000000..a3b83e2
--- /dev/null
+++ b/metron-platform/metron-hbase-server/README.md
@@ -0,0 +1,44 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# HBase Server
+
+# Introduction
+
+This project provides HBase server-side code such as coprocessors.
+
+# Coprocessors
+
+## Enrichment Coprocessor
+
+### Properties
+
+Below is the list of properties to configure the HBase enrichment coprocessor `org.apache.metron.hbase.coprocessor.EnrichmentCoprocessor`
+for writing to HBase. These configuration properties all pulled from the global config.
+
+#### `enrichment.list.hhase.provider.impl`
+
+Provider to use for obtaining the HBase table. This class implementation implements `TableProvider` and provides access to an `HTableInterface`.
+Defaults to `org.apache.metron.hbase.HTableProvider`.
+
+#### `enrichment.list.hbase.table`
+
+HBase table name for the enrichments list. Defaults to `enrichment_list`.
+
+#### `enrichment.list.hbase.cf`
+
+HBase table column family for the enrichments list. Defaults to `t`.
diff --git a/metron-platform/metron-hbase-server/pom.xml b/metron-platform/metron-hbase-server/pom.xml
new file mode 100644
index 0000000..45784d9
--- /dev/null
+++ b/metron-platform/metron-hbase-server/pom.xml
@@ -0,0 +1,326 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>metron-platform</artifactId>
+ <groupId>org.apache.metron</groupId>
+ <version>0.7.1</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>metron-hbase-server</artifactId>
+ <name>metron-hbase-server</name>
+ <url>https://metron.apache.org/</url>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ <slf4j.version>1.7.7</slf4j.version>
+ <storm.hdfs.version>0.1.2</storm.hdfs.version>
+ <guava.version>${global_hbase_guava_version}</guava.version>
+ </properties>
+
+ <dependencies>
+
+ <!-- Metron -->
+
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-enrichment-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+
+ <!-- HBase -->
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${global_hbase_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${global_hbase_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Hadoop general -->
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${global_hadoop_version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <version>${global_hadoop_version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${global_hadoop_version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${global_hadoop_version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${global_hadoop_version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Misc -->
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>1.2.17</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ <version>${global_caffeine_version}</version>
+ </dependency>
+
+ <!-- Test -->
+
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-test-utilities</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-integration-test</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <version>${global_hadoop_version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-testing-util</artifactId>
+ <version>${global_hbase_version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-data-management</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${global_shade_version}</version>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <artifactSet>
+ <excludes>
+ <!-- Exclude our SLF4J dependency that's included via the main project pom.xml -->
+ <exclude>*slf4j*</exclude>
+ </excludes>
+ </artifactSet>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <!--
+ Because we depend on other shaded jars that include dependencies like SLF4J, we have to exclude them here with the fine-grained exclusion filters.
+ Using the typical artifact exclusions will not work because it's not the original matching artifact that's introducing the dependency - it's the
+ shaded jar from upstream. Examples include: metron-statistics and metron-storm-kafka-override.
+ -->
+ <!-- Handle excluding all occurrences of SLF4 -->
+ <exclude>org/slf4j/**</exclude>
+ <exclude>META-INF/maven/org.slf4j/**</exclude>
+ <exclude>META-INF/license/LICENSE.slf4j.txt</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resources>
+ <resource>.yaml</resource>
+ <resource>LICENSE.txt</resource>
+ <resource>ASL2.0</resource>
+ <resource>NOTICE.txt</resource>
+ </resources>
+ </transformer>
+ <!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
+ <!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
+ <addHeader>false</addHeader>
+ <projectName>${project.name}</projectName>
+ </transformer-->
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass></mainClass>
+ </transformer>
+ <!-- ClassIndexTransformer needs to go LAST. For some reason it will clobber other transformers from operating when it is put first -->
+ <transformer
+ implementation="org.atteo.classindex.ClassIndexTransformer"/>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.atteo.classindex</groupId>
+ <artifactId>classindex-transformer</artifactId>
+ <version>${global_classindex_version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
diff --git a/metron-platform/metron-hbase-server/src/main/assembly/assembly.xml b/metron-platform/metron-hbase-server/src/main/assembly/assembly.xml
new file mode 100644
index 0000000..26e9f26
--- /dev/null
+++ b/metron-platform/metron-hbase-server/src/main/assembly/assembly.xml
@@ -0,0 +1,54 @@
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<assembly>
+ <id>archive</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}/src/main/config</directory>
+ <outputDirectory>config</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0644</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/src/main/scripts</directory>
+ <outputDirectory>bin</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/target</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-uber.jar</include>
+ </includes>
+ <outputDirectory>coprocessor</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/metron-platform/metron-hbase-server/src/main/java/org/apache/metron/hbase/coprocessor/EnrichmentCoprocessor.java b/metron-platform/metron-hbase-server/src/main/java/org/apache/metron/hbase/coprocessor/EnrichmentCoprocessor.java
new file mode 100644
index 0000000..ae43392
--- /dev/null
+++ b/metron-platform/metron-hbase-server/src/main/java/org/apache/metron/hbase/coprocessor/EnrichmentCoprocessor.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.hbase.coprocessor;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.CacheWriter;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.hbase.HTableProvider;
+import org.apache.metron.hbase.TableProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Handle collecting a list of enrichment coprocessors.
+ * <p>
+ * Configuration supplied via Metron global config from Zookeeper. Requires one property on startup
+ * - "zookeeperUrl" - which can be provided via HBase shell or hbase-site.xml. The typical installation
+ * mechanism provided by Metron will leverage the HBase shell.
+ * <p>
+ * <b>Note:</b> We need to be careful of our exception handling so as not to inadvertantly get our
+ * coprocessor disabled by the RegionServer. From the HBase documentation:
+ * <p>
+ * For all functions, exception handling is done as follows:
+ * <ul>
+ * <li>Exceptions of type IOException are reported back to client.</li>
+ * <li>For any other kind of exception:</li>
+ * <ul>
+ * <li>If the configuration CoprocessorHost.ABORT_ON_ERROR_KEY is set to true, then the server aborts.</li>
+ * <li>Otherwise, coprocessor is removed from the server and DoNotRetryIOException is returned to the client.</li>
+ * </ul>
+ * </ul>
+ *
+ * @see <a href="https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/coprocessor/RegionObserver.html">https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/coprocessor/RegionObserver.html</a>
+ * @see EnrichmentConfigurations Available options.
+ */
+public class EnrichmentCoprocessor extends BaseRegionObserver {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ // pass in via coprocessor config options - via hbase shell or hbase-site.xml
+ // see more here - https://hbase.apache.org/1.1/book.html#load_coprocessor_in_shell
+ public static final String ZOOKEEPER_URL = "zookeeperUrl";
+ public static final String COLUMN_QUALIFIER = "v";
+ private Cache<String, String> cache;
+ private GlobalConfigService globalConfigService;
+ private RegionCoprocessorEnvironment coprocessorEnv;
+ private Map<String, Object> globalConfig;
+
+ /**
+ * HBase requires a no-arg constructor.
+ */
+ public EnrichmentCoprocessor() {
+ }
+
+ /**
+ * Allow test dep injection.
+ */
+ public EnrichmentCoprocessor(CacheWriter<String, String> cacheWriter,
+ GlobalConfigService globalConfigService) {
+ this.cache = Caffeine.newBuilder().writer(cacheWriter).build();
+ this.globalConfigService = globalConfigService;
+ }
+
+ /**
+ * Allow test dep injection.
+ */
+ public EnrichmentCoprocessor(GlobalConfigService globalConfigService) {
+ this.globalConfigService = globalConfigService;
+ }
+
+ @Override
+ public void start(CoprocessorEnvironment ce) throws IOException {
+ LOG.info("Starting enrichment coprocessor");
+ if (ce instanceof RegionCoprocessorEnvironment) {
+ this.coprocessorEnv = (RegionCoprocessorEnvironment) ce;
+ } else {
+ throw new CoprocessorException("Enrichment coprocessor must be loaded on a table region.");
+ }
+ LOG.info("Checking if internal cache initialized");
+ if (null == this.cache) {
+ LOG.info("Cache null, initializing");
+ LOG.info("Getting global config from Zookeeper");
+ String zkUrl = getZookeeperUrl(this.coprocessorEnv.getConfiguration());
+ if (null == globalConfigService) {
+ globalConfigService = getGlobalConfigService(zkUrl);
+ }
+ globalConfig = globalConfigService.get();
+ Configuration config = this.coprocessorEnv.getConfiguration();
+ CacheWriter<String, String> cacheWriter = null;
+ try {
+ String hbaseTableProviderName = (String) globalConfig
+ .get(EnrichmentConfigurations.TABLE_PROVIDER);
+ String tableName = (String) globalConfig.get(EnrichmentConfigurations.TABLE_NAME);
+ String columnFamily = (String) globalConfig.get(EnrichmentConfigurations.COLUMN_FAMILY);
+ cacheWriter = new HBaseCacheWriter(config, TableProvider
+ .create(hbaseTableProviderName, HTableProvider::new), tableName, columnFamily,
+ COLUMN_QUALIFIER);
+ } catch (ClassNotFoundException | InstantiationException | InvocationTargetException | IllegalAccessException | NoSuchMethodException e) {
+ throw new IOException("Unable to instantiate cache writer", e);
+ }
+ this.cache = Caffeine.newBuilder().writer(cacheWriter).build();
+ LOG.info("Finished initializing cache");
+ }
+ LOG.info("Finished starting enrichment coprocessor");
+ }
+
+ private String getZookeeperUrl(Configuration config) {
+ String zkUrl = config.get(ZOOKEEPER_URL);
+ if (null == zkUrl) {
+ throw new IllegalStateException(
+ "Enrichment coprocessor requires property '" + ZOOKEEPER_URL
+ + "' to be provided at startup.");
+ }
+ return zkUrl;
+ }
+
+ private GlobalConfigService getGlobalConfigService(String zkUrl) {
+ return new GlobalConfigService() {
+ @Override
+ public Map<String, Object> get() {
+ try (CuratorFramework client = ConfigurationsUtils.getClient(zkUrl)) {
+ client.start();
+ return ConfigurationsUtils.readGlobalConfigFromZookeeper(client);
+ } catch (Exception e) {
+ throw new IllegalStateException("Unable to read global configuration from zookeeper", e);
+ }
+ }
+ };
+ }
+
+ @Override
+ public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit,
+ Durability durability) throws IOException {
+ LOG.trace("enrichment coprocessor postPut call begin");
+ try {
+ LOG.trace("Extracting enrichment type from rowkey");
+ String type = getEnrichmentType(put);
+ // Make the value json so we can add metadata at a later time, if desired.
+ final String metadata = "{}";
+ LOG.trace("Enrichment type '{}' extracted from rowkey", type);
+ addToCache(type, metadata);
+ } catch (Throwable t) {
+ LOG.warn("Exception occurred while processing Put operation in coprocessor", t);
+ // Anything other than an IOException will cause the coprocessor to be disabled.
+ throw new IOException("Error occurred while processing enrichment Put.", t);
+ }
+ LOG.trace("enrichment coprocessor postPut call complete");
+ }
+
+ private String getEnrichmentType(Put put) {
+ EnrichmentKey key = new EnrichmentKey();
+ key.fromBytes(put.getRow());
+ return key.type;
+ }
+
+ private void addToCache(String cacheKey, String value) {
+ LOG.trace("Checking if cacheKey '{}'present in cache", cacheKey);
+ // We don't want to invoke the cache writer unless we have a new key
+ if (null == cache.getIfPresent(cacheKey)) {
+ LOG.trace("cacheKey '{}' not present, adding with value='{}' to cache", cacheKey, value);
+ cache.put(cacheKey, value);
+ LOG.trace("Done adding cacheKey '{}' to cache with value='{}'", cacheKey, value);
+ }
+ }
+
+}
diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableProvider.java b/metron-platform/metron-hbase-server/src/main/java/org/apache/metron/hbase/coprocessor/GlobalConfigService.java
similarity index 65%
copy from metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableProvider.java
copy to metron-platform/metron-hbase-server/src/main/java/org/apache/metron/hbase/coprocessor/GlobalConfigService.java
index e454f04..531d813 100644
--- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableProvider.java
+++ b/metron-platform/metron-hbase-server/src/main/java/org/apache/metron/hbase/coprocessor/GlobalConfigService.java
@@ -15,17 +15,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.metron.hbase;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
+package org.apache.metron.hbase.coprocessor;
-import java.io.IOException;
+import java.util.Map;
+
+public interface GlobalConfigService {
+
+ Map<String, Object> get();
-public class HTableProvider implements TableProvider {
- @Override
- public HTableInterface getTable(Configuration config, String tableName) throws IOException {
- return new HTable(config, tableName);
- }
}
diff --git a/metron-platform/metron-hbase-server/src/main/java/org/apache/metron/hbase/coprocessor/HBaseCacheWriter.java b/metron-platform/metron-hbase-server/src/main/java/org/apache/metron/hbase/coprocessor/HBaseCacheWriter.java
new file mode 100644
index 0000000..b1bbdde
--- /dev/null
+++ b/metron-platform/metron-hbase-server/src/main/java/org/apache/metron/hbase/coprocessor/HBaseCacheWriter.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.hbase.coprocessor;
+
+import com.github.benmanes.caffeine.cache.CacheWriter;
+import com.github.benmanes.caffeine.cache.RemovalCause;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.hbase.client.HBaseClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Caffeine cache writer implementation that will write to an HBase table.
+ */
+public class HBaseCacheWriter implements CacheWriter<String, String> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private TableProvider tableProvider;
+ private final Configuration config;
+ private final String tableName;
+ private final String columnFamily;
+ private final String columnQualifier;
+
+ public HBaseCacheWriter(Configuration config, TableProvider tableProvider, String tableName,
+ String columnFamily, String columnQualifier) {
+ this.config = config;
+ this.tableProvider = tableProvider;
+ this.tableName = tableName;
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ }
+
+ /**
+ * Writes a rowkey as provided by 'key' to the configured hbase table.
+ * @param key value to use as a row key.
+ * @param value not used.
+ */
+ @Override
+ public void write(@Nonnull String key, @Nonnull String value) {
+ LOG.debug("Calling hbase cache writer with key='{}', value='{}'", key, value);
+ try (HBaseClient hbClient = new HBaseClient(this.tableProvider, this.config, this.tableName)) {
+ LOG.debug("rowKey={}, columnFamily={}, columnQualifier={}, value={}", key, columnFamily,
+ columnQualifier, value);
+ hbClient.put(key, columnFamily, columnQualifier, value);
+ LOG.debug("Done with put");
+ } catch (IOException e) {
+ throw new RuntimeException("Error writing to HBase table", e);
+ }
+ LOG.debug("Done calling hbase cache writer");
+ }
+
+ @Override
+ public void delete(@Nonnull String key, @Nullable String value, @Nonnull RemovalCause cause) {
+ // not implemented
+ }
+
+}
diff --git a/metron-platform/metron-hbase-server/src/main/scripts/load_enrichment_coprocessor.sh b/metron-platform/metron-hbase-server/src/main/scripts/load_enrichment_coprocessor.sh
new file mode 100755
index 0000000..746499f
--- /dev/null
+++ b/metron-platform/metron-hbase-server/src/main/scripts/load_enrichment_coprocessor.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+METRON_VERSION=${project.version}
+COPROCESSOR_JAR=metron-hbase-server-$METRON_VERSION-uber.jar
+
+if [ "$#" -ne 5 ]; then
+ echo "Script requires 5 params: ENRICHMENT_TABLE, HDFS_URL, HDFS_PATH, COPROCESSOR_IMPL, ZOOKEEPER_URL; Only received $#"
+ echo "Exiting"
+ exit -1
+fi
+
+ENRICHMENT_TABLE=$1
+HDFS_URL=$2
+HDFS_PATH=$3
+COPROCESSOR_IMPL=$4
+ZOOKEEPER_URL=$5
+
+echo "Altering ${ENRICHMENT_TABLE} to add coprocessor."
+echo "Executing: alter '${ENRICHMENT_TABLE}', METHOD => 'table_att', 'Coprocessor'=>'${HDFS_URL}${HDFS_PATH}/${COPROCESSOR_JAR}|${COPROCESSOR_IMPL}||zookeeperUrl=${ZOOKEEPER_URL}'"
+echo "alter '${ENRICHMENT_TABLE}', METHOD => 'table_att', 'Coprocessor'=>'${HDFS_URL}${HDFS_PATH}/${COPROCESSOR_JAR}|${COPROCESSOR_IMPL}||zookeeperUrl=${ZOOKEEPER_URL}'" | hbase shell
+echo "Done"
diff --git a/metron-platform/metron-hbase-server/src/test/java/org/apache/metron/hbase/coprocessor/EnrichmentCoprocessorIntegrationTest.java b/metron-platform/metron-hbase-server/src/test/java/org/apache/metron/hbase/coprocessor/EnrichmentCoprocessorIntegrationTest.java
new file mode 100644
index 0000000..1ff83de
--- /dev/null
+++ b/metron-platform/metron-hbase-server/src/test/java/org/apache/metron/hbase/coprocessor/EnrichmentCoprocessorIntegrationTest.java
@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.hbase.coprocessor;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.log4j.Level;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.dataloads.hbase.mr.HBaseUtil;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.hbase.helper.HelperDao;
+import org.apache.metron.integration.BaseIntegrationTest;
+import org.apache.metron.integration.ComponentRunner;
+import org.apache.metron.integration.UnableToStartException;
+import org.apache.metron.integration.components.ZKServerComponent;
+import org.apache.metron.test.utils.UnitTestHelper;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class EnrichmentCoprocessorIntegrationTest extends BaseIntegrationTest {
+
+ private static final String ENRICHMENT_TABLE = "enrichment";
+ private static final String ENRICHMENT_LIST_TABLE = "enrichment_list";
+ private static final String COLUMN_FAMILY = "c";
+
+ private static Level originalLog4jRootLoggerLevel;
+ private static java.util.logging.Level originalJavaLoggerLevel;
+ private static ZKServerComponent zookeeperComponent;
+ private static ComponentRunner componentRunner;
+ private static HBaseTestingUtility testUtil;
+ private static HTable enrichmentTable;
+ private static HTable enrichmentListTable;
+ private static Configuration hBaseConfig;
+
+ /*
+ * Test Setup
+ */
+
+ /**
+ * {
+ * "enrichment.list.hbase.provider.impl" : "org.apache.metron.hbase.HTableProvider",
+ * "enrichment.list.hbase.table" : "%TABLE_NAME%",
+ * "enrichment.list.hbase.cf" : "%COLUMN_FAMILY%"
+ * }
+ */
+ @Multiline
+ private static String globalConfig;
+
+ @BeforeClass
+ public static void setupAll() throws Exception {
+ silenceLogging();
+ // don't need the properties for anything else now, but could extract var if desired.
+ startZookeeper(new Properties());
+ globalConfig = globalConfig.replace("%TABLE_NAME%", ENRICHMENT_LIST_TABLE)
+ .replace("%COLUMN_FAMILY%", COLUMN_FAMILY);
+ uploadGlobalConfigToZK(globalConfig);
+ configureAndStartHBase();
+ addCoprocessor(enrichmentTable.getName());
+ }
+
+ /**
+ * log4j and java logging set to ERROR, SEVERE respectively.
+ */
+ private static void silenceLogging() {
+ originalLog4jRootLoggerLevel = UnitTestHelper.getLog4jLevel();
+ originalJavaLoggerLevel = UnitTestHelper.getJavaLoggingLevel();
+ UnitTestHelper.setLog4jLevel(Level.ERROR);
+ // uncomment below for finer-grained logging
+ /*
+ UnitTestHelper.setLog4jLevel(EnrichmentCoprocessor.class, Level.DEBUG);
+ UnitTestHelper.setLog4jLevel(HBaseCacheWriter.class, Level.DEBUG);
+ */
+ UnitTestHelper.setJavaLoggingLevel(java.util.logging.Level.SEVERE);
+ }
+
+ /**
+ * Starts zookeeper.
+ * @param properties the zk setup will modify properties arg with the setup detail.
+ * @throws UnableToStartException zk fails to start.
+ */
+ private static void startZookeeper(Properties properties) throws UnableToStartException {
+ zookeeperComponent = getZKServerComponent(properties);
+ componentRunner = new ComponentRunner.Builder()
+ .withComponent("zk", zookeeperComponent)
+ .withMillisecondsBetweenAttempts(15000)
+ .withNumRetries(10)
+ .build();
+ componentRunner.start();
+ }
+
+ private static void uploadGlobalConfigToZK(String config) throws Exception {
+ ConfigurationsUtils.writeGlobalConfigToZookeeper(config.getBytes(StandardCharsets.UTF_8),
+ zookeeperComponent.getConnectionString());
+ }
+
+ /**
+ * Start HBase.
+ * Create enrichment and enrichment list tables.
+ */
+ private static void configureAndStartHBase() throws Exception {
+ Configuration extraConfig = new Configuration();
+ extraConfig.set(EnrichmentCoprocessor.ZOOKEEPER_URL, zookeeperComponent.getConnectionString());
+ Map.Entry<HBaseTestingUtility, Configuration> kv = HBaseUtil.INSTANCE.create(true, extraConfig);
+ testUtil = kv.getKey();
+ hBaseConfig = kv.getValue();
+ enrichmentTable = testUtil.createTable(Bytes.toBytes(ENRICHMENT_TABLE), Bytes.toBytes(
+ COLUMN_FAMILY));
+ enrichmentListTable = testUtil
+ .createTable(Bytes.toBytes(ENRICHMENT_LIST_TABLE), Bytes.toBytes(COLUMN_FAMILY));
+
+ for (Result r : enrichmentTable.getScanner(Bytes.toBytes(COLUMN_FAMILY))) {
+ Delete d = new Delete(r.getRow());
+ enrichmentTable.delete(d);
+ }
+ for (Result r : enrichmentListTable.getScanner(Bytes.toBytes(COLUMN_FAMILY))) {
+ Delete d = new Delete(r.getRow());
+ enrichmentListTable.delete(d);
+ }
+ }
+
+ private static void addCoprocessor(TableName tableName) throws IOException {
+ // https://hbase.apache.org/1.1/book.html#cp_loading
+ Admin hbaseAdmin = testUtil.getConnection().getAdmin();
+ hbaseAdmin.disableTable(tableName);
+ HTableDescriptor htd = new HTableDescriptor(tableName);
+ htd.addFamily(new HColumnDescriptor(COLUMN_FAMILY));
+ htd.addCoprocessor(EnrichmentCoprocessor.class.getCanonicalName());
+ hbaseAdmin.modifyTable(tableName, htd);
+ hbaseAdmin.enableTable(tableName);
+ }
+
+ @AfterClass
+ public static void teardown() throws Exception {
+ HBaseUtil.INSTANCE.teardown(testUtil);
+ componentRunner.stop();
+ resetLogging();
+ }
+
+ private static void resetLogging() {
+ UnitTestHelper.setLog4jLevel(originalLog4jRootLoggerLevel);
+ UnitTestHelper.setJavaLoggingLevel(originalJavaLoggerLevel);
+ }
+
+ /*
+ * Tests
+ */
+
+ @Test
+ public void enrichments_loaded_in_list_table() throws Exception {
+ // indicator, type
+ Map<String, String> enrichments = new HashMap<String, String>() {{
+ put("111", "foo");
+ put("222", "foo");
+ put("333", "bar");
+ put("444", "bar");
+ put("555", "baz");
+ put("666", "baz");
+ }};
+ Set<String> expectedEnrichmentTypes = new HashSet<>();
+ for (Map.Entry<String, String> enrichKV : enrichments.entrySet()) {
+ String indicator = enrichKV.getKey();
+ String type = enrichKV.getValue();
+ expectedEnrichmentTypes.add(type);
+ HelperDao.insertRecord(enrichmentTable, new EnrichmentKey(type, indicator), COLUMN_FAMILY,
+ "{ \"apache\" : \"metron\" }");
+ }
+ List<String> enrichmentsList = HelperDao.readRecords(enrichmentListTable);
+ assertThat(new HashSet<String>(enrichmentsList), equalTo(expectedEnrichmentTypes));
+ }
+
+}
diff --git a/metron-platform/metron-hbase-server/src/test/java/org/apache/metron/hbase/coprocessor/EnrichmentCoprocessorTest.java b/metron-platform/metron-hbase-server/src/test/java/org/apache/metron/hbase/coprocessor/EnrichmentCoprocessorTest.java
new file mode 100644
index 0000000..19bd4b4
--- /dev/null
+++ b/metron-platform/metron-hbase-server/src/test/java/org/apache/metron/hbase/coprocessor/EnrichmentCoprocessorTest.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.hbase.coprocessor;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.junit.Assert.assertThat;
+import static org.mockito.BDDMockito.willAnswer;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.github.benmanes.caffeine.cache.CacheWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.hbase.TableProvider;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+public class EnrichmentCoprocessorTest {
+
+ @Mock
+ private CacheWriter<String, String> cacheWriter;
+ @Mock
+ private RegionCoprocessorEnvironment copEnv;
+ @Mock
+ private ObserverContext<RegionCoprocessorEnvironment> observerContext;
+ private EnrichmentCoprocessor cop;
+ @Mock
+ private GlobalConfigService globalConfigService;
+ private Configuration config;
+ private static boolean instantiatedCustomTableProvider;
+
+ @Before
+ public void setup() {
+ MockitoAnnotations.initMocks(this);
+ cop = new EnrichmentCoprocessor(cacheWriter, globalConfigService);
+ config = HBaseConfiguration.create();
+ config.set(EnrichmentCoprocessor.ZOOKEEPER_URL, "foobar");
+ when(copEnv.getConfiguration()).thenReturn(config);
+ instantiatedCustomTableProvider = false;
+ }
+
+ @Test
+ public void cache_writes_only_on_first_cache_miss() throws Exception {
+ cop.start(copEnv);
+ String[] enrichTypes = new String[]{"foo", "bar", "baz", "metron"};
+ final int putsPerType = 3;
+ Map<String, List<Put>> putsByType = simulateMultiplePutsPerType(putsPerType, enrichTypes);
+ int totalPuts = 0;
+ for (Map.Entry<String, List<Put>> entry : putsByType.entrySet()) {
+ String type = entry.getKey();
+ List<Put> puts = entry.getValue();
+ for (Put put : puts) {
+ cop.postPut(observerContext, put, null, null);
+ verify(cacheWriter, times(1)).write(eq(type), eq("{}"));
+ totalPuts++;
+ }
+ }
+ assertThat(totalPuts, equalTo(enrichTypes.length * putsPerType));
+ }
+
+ /**
+ * Generate a list of 'count' puts for each type in 'types'.
+ *
+ * @param count Number of puts to create per type
+ * @param types List of types to create the puts for.
+ * @return Map of types to a List of size 'count' puts
+ */
+ private Map<String, List<Put>> simulateMultiplePutsPerType(int count, String... types) {
+ Map<String, List<Put>> putsByType = new HashMap<>();
+ for (String type : types) {
+ List<Put> puts = putsByType.getOrDefault(type, new ArrayList<>());
+ for (int i = 0; i < count; i++) {
+ EnrichmentKey ek = new EnrichmentKey(type, String.valueOf(i));
+ puts.add(new Put(ek.toBytes()));
+ putsByType.put(type, puts);
+ }
+ }
+ return putsByType;
+ }
+
+ public static class TestTableProvider implements TableProvider {
+
+ public TestTableProvider() {
+ instantiatedCustomTableProvider = true;
+ }
+
+ @Override
+ public HTableInterface getTable(Configuration config, String tableName) throws IOException {
+ return null; // not used for instantiation test
+ }
+ }
+
+ @Test
+ public void creates_tableprovider_from_config_property() throws Exception {
+ cop = new EnrichmentCoprocessor(globalConfigService);
+ Map<String, Object> globalConfig = new HashMap<String, Object>() {{
+ put(EnrichmentConfigurations.TABLE_PROVIDER, TestTableProvider.class.getName());
+ }};
+ when(globalConfigService.get()).thenReturn(globalConfig);
+ cop.start(copEnv);
+ assertThat(instantiatedCustomTableProvider, equalTo(true));
+ }
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void bad_enrichment_key_exceptions_thrown_as_IOException() throws Exception {
+ thrown.expect(IOException.class);
+ thrown.expectMessage("Error occurred while processing enrichment Put.");
+ thrown.expectCause(instanceOf(RuntimeException.class));
+ cop.start(copEnv);
+ cop.postPut(observerContext, new Put("foo".getBytes()), null, null);
+ }
+
+ @Test
+ public void general_exceptions_thrown_as_IOException() throws Exception {
+ Throwable cause = new Throwable("Bad things happened.");
+ thrown.expect(IOException.class);
+ thrown.expectMessage("Error occurred while processing enrichment Put.");
+ thrown.expectCause(equalTo(cause));
+ // strictly speaking, this is a checked exception not in the api for CacheWriter, but it allows
+ // us to test catching all Throwable types
+ willAnswer(i -> {
+ throw cause;
+ }).given(cacheWriter).write(any(), any());
+ cop.start(copEnv);
+ EnrichmentKey ek = new EnrichmentKey("foo", "bar");
+ cop.postPut(observerContext, new Put(ek.toBytes()), null, null);
+ }
+
+}
diff --git a/metron-platform/metron-hbase-server/src/test/java/org/apache/metron/hbase/helper/HelperDao.java b/metron-platform/metron-hbase-server/src/test/java/org/apache/metron/hbase/helper/HelperDao.java
new file mode 100644
index 0000000..2aae9e7
--- /dev/null
+++ b/metron-platform/metron-hbase-server/src/test/java/org/apache/metron/hbase/helper/HelperDao.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.hbase.helper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.enrichment.converter.EnrichmentConverter;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.converter.EnrichmentValue;
+
+public class HelperDao {
+
+ public static void insertRecord(Table table, EnrichmentKey key, String cf, String value)
+ throws IOException {
+ Put put = createPut(key, cf, value);
+ table.put(put);
+ }
+
+ private static Put createPut(EnrichmentKey rowKey, String cf, String value) throws IOException {
+ return new EnrichmentConverter().toPut(cf, rowKey,
+ new EnrichmentValue(JSONUtils.INSTANCE.load(value, JSONUtils.MAP_SUPPLIER)));
+ }
+
+ public static List<String> readRecords(Table table) throws Exception {
+ Scan scan = new Scan();
+ ResultScanner scanner = table.getScanner(scan);
+ List<String> rows = new ArrayList<>();
+ for (Result r = scanner.next(); r != null; r = scanner.next()) {
+ rows.add(Bytes.toString(r.getRow()));
+ }
+ return rows;
+ }
+
+}
diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableProvider.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableProvider.java
index e454f04..a323beb 100644
--- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableProvider.java
+++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/HTableProvider.java
@@ -17,12 +17,11 @@
*/
package org.apache.metron.hbase;
+import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTableInterface;
-import java.io.IOException;
-
public class HTableProvider implements TableProvider {
@Override
public HTableInterface getTable(Configuration config, String tableName) throws IOException {
diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/TableProvider.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/TableProvider.java
index 1804697..8511f75 100644
--- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/TableProvider.java
+++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/TableProvider.java
@@ -27,6 +27,14 @@ import java.util.function.Supplier;
public interface TableProvider extends Serializable {
HTableInterface getTable(Configuration config, String tableName) throws IOException;
+
+ /**
+ * Factory method that creates TableProviders.
+ *
+ * @param impl attempt to create this type of TableProvider
+ * @param defaultSupplier provides default implementation if impl is null
+ * @return New table provider
+ */
static TableProvider create(String impl, Supplier<TableProvider> defaultSupplier) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
if(impl == null) {
return defaultSupplier.get();
diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/client/HBaseClient.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/client/HBaseClient.java
index 9ddf9b7..51fc2c6 100644
--- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/client/HBaseClient.java
+++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/client/HBaseClient.java
@@ -35,6 +35,9 @@ import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.metron.hbase.TableProvider;
import org.apache.metron.hbase.bolt.mapper.ColumnList;
import org.apache.metron.hbase.bolt.mapper.HBaseProjectionCriteria;
@@ -300,4 +303,35 @@ public class HBaseClient implements Closeable {
}
return tableName;
}
+
+ /**
+ * Puts a record into the configured HBase table synchronously (not batched).
+ */
+ public void put(String rowKey, String columnFamily, String columnQualifier, String value)
+ throws IOException {
+ Put put = new Put(Bytes.toBytes(rowKey));
+ put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(columnQualifier),
+ Bytes.toBytes(value));
+ table.put(put);
+ }
+
+ /**
+ * Scans an entire table returning all row keys as a List of Strings.
+ *
+ * <p>
+ * <b>**WARNING**:</b> Do not use this method unless you're absolutely crystal clear about the performance
+ * impact. Doing full table scans in HBase can adversely impact performance.
+ *
+ * @return List of all row keys as Strings for this table.
+ */
+ public List<String> readRecords() throws IOException {
+ Scan scan = new Scan();
+ ResultScanner scanner = table.getScanner(scan);
+ List<String> rows = new ArrayList<>();
+ for (Result r = scanner.next(); r != null; r = scanner.next()) {
+ rows.add(Bytes.toString(r.getRow()));
+ }
+ return rows;
+ }
+
}
diff --git a/metron-platform/metron-hbase/src/test/resources/log4j.properties b/metron-platform/metron-hbase/src/test/resources/log4j.properties
index 70be8ae..169718f 100644
--- a/metron-platform/metron-hbase/src/test/resources/log4j.properties
+++ b/metron-platform/metron-hbase/src/test/resources/log4j.properties
@@ -1,5 +1,3 @@
-#
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,8 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
-#
# Root logger option
log4j.rootLogger=ERROR, stdout
@@ -26,3 +22,6 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+log4j.appender.stdout.filter.1=org.apache.log4j.varia.StringMatchFilter
+log4j.appender.stdout.filter.1.StringToMatch=interrupted
+log4j.appender.stdout.filter.1.AcceptOnMatch=false
diff --git a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/UnitTestHelper.java b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/UnitTestHelper.java
index 5a705e3..37d122c 100644
--- a/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/UnitTestHelper.java
+++ b/metron-platform/metron-test-utilities/src/main/java/org/apache/metron/test/utils/UnitTestHelper.java
@@ -17,11 +17,7 @@
*/
package org.apache.metron.test.utils;
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
-import org.junit.Assert;
+import static java.lang.String.format;
import java.io.File;
import java.io.IOException;
@@ -33,8 +29,11 @@ import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Set;
import java.util.Stack;
-
-import static java.lang.String.format;
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+import org.junit.Assert;
public class UnitTestHelper {
public static String findDir(String name) {
@@ -96,6 +95,24 @@ public class UnitTestHelper {
logger.setLevel(level);
}
+ /**
+ * Root logger.
+ * @param level level for root logger
+ */
+ public static void setLog4jLevel(Level level) {
+ Logger logger = Logger.getRootLogger();
+ logger.setLevel(level);
+ }
+
+ /**
+ * Root logger.
+ * @param level level for root logger
+ */
+ public static Level getLog4jLevel() {
+ Logger rootLogger = Logger.getRootLogger();
+ return rootLogger.getLevel();
+ }
+
public static Level getLog4jLevel(Class clazz) {
Logger logger = Logger.getLogger(clazz);
return logger.getLevel();
diff --git a/metron-platform/pom.xml b/metron-platform/pom.xml
index 6958dcf..af563cf 100644
--- a/metron-platform/pom.xml
+++ b/metron-platform/pom.xml
@@ -61,6 +61,7 @@
<module>metron-storm-kafka-override</module>
<module>metron-zookeeper</module>
<module>metron-parsing</module>
+ <module>metron-hbase-server</module>
</modules>
<dependencies>
<dependency>