You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/05/10 16:53:54 UTC
[2/5] incubator-metron git commit: METRON-141: The ability to do
threat triage closes apache/incubator-metron#108
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/bash/Whois_CSV_to_JSON.py
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/Whois_CSV_to_JSON.py b/metron-platform/metron-data-management/src/main/bash/Whois_CSV_to_JSON.py
deleted file mode 100755
index 2091418..0000000
--- a/metron-platform/metron-data-management/src/main/bash/Whois_CSV_to_JSON.py
+++ /dev/null
@@ -1,208 +0,0 @@
-#!/usr/bin/python
-
-"""
-Copyright 2014 Cisco Systems, Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import sys
-import os
-import csv
-import json
-import multiprocessing
-import logging
-logging.basicConfig(level=logging.DEBUG)
-
-
-def is_field_excluded(fieldname=None):
- """
- Checks to see if a field name is a member of a list of names to exclude. Modify to suit your own list.
-
- :param fieldname: A string representing a field name
- :return: True or False
- """
- import re
-
- # List of fields names to exclude
- excluded_fields = [
- 'Audit_auditUpdatedDate',
- #'domainName'
- ]
-
- if fieldname in excluded_fields:
- return True
-
- # Regexes to match for exclusion
- excluded_regexes = [
- ['_rawText$', re.IGNORECASE],
- ]
-
- for regex in excluded_regexes:
- if re.search(regex[0], fieldname, regex[1]):
- return True
-
- return False
-
-
-def process_csv(in_filename, out_filename):
- """
- Processes a CSV file of WHOIS data and converts each line to a JSON element, skipping specific fields that
- are not deemed necessary (domainName, *_rawText, Audit_auditUpdatedDate)
-
- :param in_filename: Input CSV filename with full path
- :param out_filename: Output JSON filename with full path
- :return: None
- """
- if out_filename:
- out_fh = open(out_filename, 'wb')
- logging.debug('%s: Converting %s to %s' % (multiprocessing.current_process().name, in_filename, out_filename))
- else:
- logging.debug('%s: Analyzing %s' % (multiprocessing.current_process().name, in_filename))
-
- with open(in_filename, 'rb') as f:
- reader = csv.DictReader(f, delimiter=',', quotechar='"')
- line_num = 0
- try:
- for row in reader:
- line_num += 1
- try:
- if out_filename:
- # json conversion and output
- new_row = {}
- for field in reader.fieldnames:
- # fields we don't want include these + anything with rawText
- #if field not in ['Audit_auditUpdatedDate', 'domainName'] and not field.endswith('_rawText'):
- if not is_field_excluded(field):
- new_row[field] = row.get(field)
- json.dump(new_row, out_fh)
- out_fh.write('\n')
- else:
- # analysis .. check to be sure fileheader and csv row counts match
- if len(row) != len(reader.fieldnames):
- raise Exception('Field count mismatch: row: %s / fields: %s' % (len(row), len(reader.fieldnames)))
- except Exception, e:
- logging.warn("Error with file %s, line %s: %s" % (in_filename, line_num, e))
-
- if not out_filename:
- logging.info('Analyzed %s: OK' % in_filename)
- except Exception, e:
- logging.warn(e)
-
- out_fh.close()
-
-
-##-------------------------------------------------------------------------
-
-def process_files(source_dir, output_dir, max_processes=10, overwrite=False):
- """
- Generates a multiprocessing.Pool() queue with a list of input and output files to be processed with processCSV.
- Files are added by walking the source_dir and adding any file with a CSV extension. Output is placed into a single
- directory for processing. Output filenames are generated using the first part of the directory name so a file
- named source_dir/com/1.csv would become outputDir/com_1.json
-
- :param source_dir: Source directory of CSV files
- :param output_dir: Output directory for resultant JSON files
- :param max_processes: Maximum number of processes run
- :return:
- """
- logging.info("Processing Whois files from %s" % source_dir)
-
- if output_dir and not os.path.exists(output_dir):
- logging.debug("Creating output directory %s" % output_dir)
- os.makedirs(output_dir)
-
- logging.info("Starting %s pool workers" % max_processes)
-
- if sys.version.startswith('2.6'):
- # no maxtaskperchild in 2.6
- pool = multiprocessing.Pool(processes=max_processes)
- else:
- pool = multiprocessing.Pool(processes=max_processes, maxtasksperchild=4)
-
- filecount = 0
- for dirname, dirnames, filenames in os.walk(source_dir):
- for filename in filenames:
- if filename.endswith('.csv'):
- # output files go to outputDir and are named using the last subdirectory from the dirname
- if output_dir:
- out_filename = filename.replace('csv', 'json')
- out_filename = os.path.join(output_dir, '%s_%s' % (os.path.split(dirname)[-1], out_filename))
-
- # if file does not exist or if overwrite is true, add file process to the pool
- if not os.path.isfile(out_filename) or overwrite:
- pool.apply_async(process_csv, args=(os.path.join(dirname, filename), out_filename))
- filecount += 1
- else:
- logging.info("Skipping %s, %s exists and overwrite is false" % (filename, out_filename))
- else:
- # no outputdir so we just analyze the files
- pool.apply_async(process_csv, args=(os.path.join(dirname, filename), None))
- filecount += 1
-
- try:
- pool.close()
- logging.info("Starting activities on %s CSV files" % filecount)
- pool.join()
- except KeyboardInterrupt:
- logging.info("Aborting")
- pool.terminate()
-
- logging.info("Completed")
-
-
-##-------------------------------------------------------------------------
-
-if __name__ == "__main__":
-
- max_cpu = multiprocessing.cpu_count()
-
- from optparse import OptionParser
- parser = OptionParser()
- parser.add_option('-s', '--source', dest='source_dir', action='store',
- help='Source directory to walk for CSV files')
- parser.add_option('-o', '--output', dest='out_dir', action='store',
- help='Output directory for JSON files')
- parser.add_option('-O', '--overwrite', dest='overwrite', action='store_true',
- help='Overwrite existing files in output directory')
- parser.add_option('-p', '--processes', dest='max_processes', action='store', default=max_cpu, type='int',
- help='Max number of processes to spawn')
- parser.add_option('-a', '--analyze', dest='analyze', action='store_true',
- help='Analyze CSV files for validity, no file output')
- parser.add_option('-d', '--debug', dest='debug', action='store_true',
- help='Enable debug messages')
-
- (options, args) = parser.parse_args()
-
- if not options.source_dir:
- logging.error("Source directory required")
- sys.exit(-1)
-
- if not options.out_dir or options.analyze:
- out_dir = None
- elif not options.out_dir:
- logging.error("Ouput directory or analysis option required")
- sys.exit(-1)
- else:
- out_dir = options.out_dir
-
- if options.max_processes > max_cpu:
- logging.warn('Max Processes (%s) is greater than available Processors (%s)' % (options.max_processes, max_cpu))
-
- if options.debug:
- # enable debug level and multiprocessing debugging
- logging.basicConfig(level=logging.DEBUG)
- multiprocessing.log_to_stderr(logging.DEBUG)
-
- process_files(options.source_dir, options.out_dir, options.max_processes, options.overwrite)
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/bash/flatfile_loader.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/flatfile_loader.sh b/metron-platform/metron-data-management/src/main/bash/flatfile_loader.sh
deleted file mode 100755
index 03fd641..0000000
--- a/metron-platform/metron-data-management/src/main/bash/flatfile_loader.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
-[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
-
-# Autodetect JAVA_HOME if not defined
-if [ -e /usr/libexec/bigtop-detect-javahome ]; then
- . /usr/libexec/bigtop-detect-javahome
-elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
- . /usr/lib/bigtop-utils/bigtop-detect-javahome
-fi
-
-export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
-CP=/usr/metron/0.1BETA/lib/metron-data-management-0.1BETA.jar:/usr/metron/0.1BETA/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath`
-HADOOP_CLASSPATH=$(echo $CP )
-for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
- if [ -f $jar ];then
- LIBJARS="$jar,$LIBJARS"
- fi
-done
-export HADOOP_CLASSPATH
-export METRON_VERSION=${project.version}
-export METRON_HOME=/usr/metron/$METRON_VERSION
-export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
-hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.nonbulk.flatfile.SimpleEnrichmentFlatFileLoader "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/bash/prune_elasticsearch_indices.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/prune_elasticsearch_indices.sh b/metron-platform/metron-data-management/src/main/bash/prune_elasticsearch_indices.sh
deleted file mode 100644
index c3f1d05..0000000
--- a/metron-platform/metron-data-management/src/main/bash/prune_elasticsearch_indices.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}.jar org.apache.metron.dataloads.bulk.ElasticsearchDataPrunerRunner "$@"
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/bash/prune_hdfs_files.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/prune_hdfs_files.sh b/metron-platform/metron-data-management/src/main/bash/prune_hdfs_files.sh
deleted file mode 100644
index b8d9372..0000000
--- a/metron-platform/metron-data-management/src/main/bash/prune_hdfs_files.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}.jar org.apache.metron.dataloads.bulk.HDFSDataPruner "$@"
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_load.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_load.sh b/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_load.sh
deleted file mode 100755
index 865d0ad..0000000
--- a/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_load.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
-[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
-
-# Autodetect JAVA_HOME if not defined
-if [ -e /usr/libexec/bigtop-detect-javahome ]; then
- . /usr/libexec/bigtop-detect-javahome
-elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
- . /usr/lib/bigtop-utils/bigtop-detect-javahome
-fi
-
-export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
-HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
-for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
- if [ -f $jar ];then
- LIBJARS="$jar,$LIBJARS"
- fi
-done
-export HADOOP_CLASSPATH
-export METRON_VERSION=${project.version}
-export METRON_HOME=/usr/metron/$METRON_VERSION
-export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
-hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.bulk.ThreatIntelBulkLoader -libjars ${LIBJARS} "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_prune.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_prune.sh b/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_prune.sh
deleted file mode 100755
index 6156027..0000000
--- a/metron-platform/metron-data-management/src/main/bash/threatintel_bulk_prune.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
-[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
-
-# Autodetect JAVA_HOME if not defined
-if [ -e /usr/libexec/bigtop-detect-javahome ]; then
- . /usr/libexec/bigtop-detect-javahome
-elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
- . /usr/lib/bigtop-utils/bigtop-detect-javahome
-fi
-
-export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
-HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
-for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
- if [ -f $jar ];then
- LIBJARS="$jar,$LIBJARS"
- fi
-done
-export HADOOP_CLASSPATH
-export METRON_VERSION=${project.version}
-export METRON_HOME=/usr/metron/$METRON_VERSION
-export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
-hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.bulk.LeastRecentlyUsedPruner -libjars ${LIBJARS} "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/bash/threatintel_taxii_load.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/bash/threatintel_taxii_load.sh b/metron-platform/metron-data-management/src/main/bash/threatintel_taxii_load.sh
deleted file mode 100755
index 23d09ba..0000000
--- a/metron-platform/metron-data-management/src/main/bash/threatintel_taxii_load.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
-[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
-
-# Autodetect JAVA_HOME if not defined
-if [ -e /usr/libexec/bigtop-detect-javahome ]; then
- . /usr/libexec/bigtop-detect-javahome
-elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
- . /usr/lib/bigtop-utils/bigtop-detect-javahome
-fi
-
-export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
-CP=/usr/metron/0.1BETA/lib/metron-data-management-0.1BETA.jar:/usr/metron/0.1BETA/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath`
-HADOOP_CLASSPATH=$(echo $CP )
-for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
- if [ -f $jar ];then
- LIBJARS="$jar,$LIBJARS"
- fi
-done
-export HADOOP_CLASSPATH
-export METRON_VERSION=${project.version}
-export METRON_HOME=/usr/metron/$METRON_VERSION
-export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
-hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.nonbulk.taxii.TaxiiLoader "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
index 2e63ef2..4e2da61 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.metron.dataloads.extractor.ExtractorHandler;
import org.apache.metron.dataloads.hbase.mr.BulkLoadMapper;
-import org.apache.metron.common.configuration.EnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig;
import org.apache.metron.enrichment.converter.HbaseConverter;
import org.apache.metron.enrichment.converter.EnrichmentConverter;
import org.apache.metron.common.utils.JSONUtils;
@@ -238,10 +238,10 @@ public class ThreatIntelBulkLoader {
if(BulkLoadOptions.CONVERTER.has(cli)) {
converterClass = BulkLoadOptions.CONVERTER.get(cli);
}
- EnrichmentConfig enrichmentConfig = null;
+ SensorEnrichmentUpdateConfig sensorEnrichmentUpdateConfig = null;
if(BulkLoadOptions.ENRICHMENT_CONFIG.has(cli)) {
- enrichmentConfig = JSONUtils.INSTANCE.load( new File(BulkLoadOptions.ENRICHMENT_CONFIG.get(cli))
- , EnrichmentConfig.class
+ sensorEnrichmentUpdateConfig = JSONUtils.INSTANCE.load( new File(BulkLoadOptions.ENRICHMENT_CONFIG.get(cli))
+ , SensorEnrichmentUpdateConfig.class
);
}
@@ -252,8 +252,8 @@ public class ThreatIntelBulkLoader {
if(!jobRet) {
System.exit(1);
}
- if(enrichmentConfig != null) {
- enrichmentConfig.updateSensorConfigs();
+ if(sensorEnrichmentUpdateConfig != null) {
+ sensorEnrichmentUpdateConfig.updateSensorConfigs();
}
System.exit(0);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoader.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoader.java
index cbd3beb..0c7501a 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoader.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoader.java
@@ -31,7 +31,7 @@ import org.apache.log4j.PropertyConfigurator;
import org.apache.metron.dataloads.extractor.Extractor;
import org.apache.metron.dataloads.extractor.ExtractorHandler;
import org.apache.metron.dataloads.extractor.inputformat.WholeFileFormat;
-import org.apache.metron.common.configuration.EnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig;
import org.apache.metron.hbase.HTableProvider;
import org.apache.metron.enrichment.converter.HbaseConverter;
import org.apache.metron.enrichment.converter.EnrichmentConverter;
@@ -239,10 +239,10 @@ public class SimpleEnrichmentFlatFileLoader {
);
boolean lineByLine = !handler.getInputFormatHandler().getClass().equals(WholeFileFormat.class);
Extractor e = handler.getExtractor();
- EnrichmentConfig enrichmentConfig = null;
+ SensorEnrichmentUpdateConfig sensorEnrichmentUpdateConfig = null;
if(LoadOptions.ENRICHMENT_CONFIG.has(cli)) {
- enrichmentConfig = JSONUtils.INSTANCE.load( new File(LoadOptions.ENRICHMENT_CONFIG.get(cli))
- , EnrichmentConfig.class
+ sensorEnrichmentUpdateConfig = JSONUtils.INSTANCE.load( new File(LoadOptions.ENRICHMENT_CONFIG.get(cli))
+ , SensorEnrichmentUpdateConfig.class
);
}
HbaseConverter converter = new EnrichmentConverter();
@@ -254,8 +254,8 @@ public class SimpleEnrichmentFlatFileLoader {
for (File f : inputFiles) {
loader.loadFile(f, e, table, LoadOptions.HBASE_CF.get(cli), converter, lineByLine);
}
- if(enrichmentConfig != null) {
- enrichmentConfig.updateSensorConfigs();
+ if(sensorEnrichmentUpdateConfig != null) {
+ sensorEnrichmentUpdateConfig.updateSensorConfigs();
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
index df803c1..689a08f 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
@@ -31,7 +31,7 @@ import org.apache.log4j.PropertyConfigurator;
import org.apache.metron.dataloads.extractor.Extractor;
import org.apache.metron.dataloads.extractor.ExtractorHandler;
import org.apache.metron.dataloads.extractor.stix.StixExtractor;
-import org.apache.metron.common.configuration.EnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig;
import org.apache.metron.common.utils.JSONUtils;
import javax.annotation.Nullable;
@@ -176,12 +176,12 @@ public class TaxiiLoader {
}
ExtractorHandler handler = ExtractorHandler.load(FileUtils.readFileToString(new File(TaxiiOptions.EXTRACTOR_CONFIG.get(cli))));
Extractor e = handler.getExtractor();
- EnrichmentConfig enrichmentConfig = null;
+ SensorEnrichmentUpdateConfig sensorEnrichmentUpdateConfig = null;
if(TaxiiOptions.ENRICHMENT_CONFIG.has(cli)) {
- enrichmentConfig = JSONUtils.INSTANCE.load( new File(TaxiiOptions.ENRICHMENT_CONFIG.get(cli))
- , EnrichmentConfig.class
+ sensorEnrichmentUpdateConfig = JSONUtils.INSTANCE.load( new File(TaxiiOptions.ENRICHMENT_CONFIG.get(cli))
+ , SensorEnrichmentUpdateConfig.class
);
- enrichmentConfig.updateSensorConfigs();
+ sensorEnrichmentUpdateConfig.updateSensorConfigs();
}
Timer timer = new Timer();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/scripts/Whois_CSV_to_JSON.py
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/scripts/Whois_CSV_to_JSON.py b/metron-platform/metron-data-management/src/main/scripts/Whois_CSV_to_JSON.py
new file mode 100755
index 0000000..2091418
--- /dev/null
+++ b/metron-platform/metron-data-management/src/main/scripts/Whois_CSV_to_JSON.py
@@ -0,0 +1,208 @@
+#!/usr/bin/python
+
+"""
+Copyright 2014 Cisco Systems, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import sys
+import os
+import csv
+import json
+import multiprocessing
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+
+def is_field_excluded(fieldname=None):
+ """
+ Checks to see if a field name is a member of a list of names to exclude. Modify to suit your own list.
+
+ :param fieldname: A string representing a field name
+ :return: True or False
+ """
+ import re
+
+ # List of fields names to exclude
+ excluded_fields = [
+ 'Audit_auditUpdatedDate',
+ #'domainName'
+ ]
+
+ if fieldname in excluded_fields:
+ return True
+
+ # Regexes to match for exclusion
+ excluded_regexes = [
+ ['_rawText$', re.IGNORECASE],
+ ]
+
+ for regex in excluded_regexes:
+ if re.search(regex[0], fieldname, regex[1]):
+ return True
+
+ return False
+
+
+def process_csv(in_filename, out_filename):
+ """
+ Processes a CSV file of WHOIS data and converts each line to a JSON element, skipping specific fields that
+ are not deemed necessary (domainName, *_rawText, Audit_auditUpdatedDate)
+
+ :param in_filename: Input CSV filename with full path
+ :param out_filename: Output JSON filename with full path
+ :return: None
+ """
+ if out_filename:
+ out_fh = open(out_filename, 'wb')
+ logging.debug('%s: Converting %s to %s' % (multiprocessing.current_process().name, in_filename, out_filename))
+ else:
+ logging.debug('%s: Analyzing %s' % (multiprocessing.current_process().name, in_filename))
+
+ with open(in_filename, 'rb') as f:
+ reader = csv.DictReader(f, delimiter=',', quotechar='"')
+ line_num = 0
+ try:
+ for row in reader:
+ line_num += 1
+ try:
+ if out_filename:
+ # json conversion and output
+ new_row = {}
+ for field in reader.fieldnames:
+ # fields we don't want include these + anything with rawText
+ #if field not in ['Audit_auditUpdatedDate', 'domainName'] and not field.endswith('_rawText'):
+ if not is_field_excluded(field):
+ new_row[field] = row.get(field)
+ json.dump(new_row, out_fh)
+ out_fh.write('\n')
+ else:
+ # analysis .. check to be sure fileheader and csv row counts match
+ if len(row) != len(reader.fieldnames):
+ raise Exception('Field count mismatch: row: %s / fields: %s' % (len(row), len(reader.fieldnames)))
+ except Exception, e:
+ logging.warn("Error with file %s, line %s: %s" % (in_filename, line_num, e))
+
+ if not out_filename:
+ logging.info('Analyzed %s: OK' % in_filename)
+ except Exception, e:
+ logging.warn(e)
+
+ out_fh.close()
+
+
+##-------------------------------------------------------------------------
+
+def process_files(source_dir, output_dir, max_processes=10, overwrite=False):
+ """
+ Generates a multiprocessing.Pool() queue with a list of input and output files to be processed with processCSV.
+ Files are added by walking the source_dir and adding any file with a CSV extension. Output is placed into a single
+ directory for processing. Output filenames are generated using the first part of the directory name so a file
+ named source_dir/com/1.csv would become outputDir/com_1.json
+
+ :param source_dir: Source directory of CSV files
+ :param output_dir: Output directory for resultant JSON files
+ :param max_processes: Maximum number of processes run
+ :return:
+ """
+ logging.info("Processing Whois files from %s" % source_dir)
+
+ if output_dir and not os.path.exists(output_dir):
+ logging.debug("Creating output directory %s" % output_dir)
+ os.makedirs(output_dir)
+
+ logging.info("Starting %s pool workers" % max_processes)
+
+ if sys.version.startswith('2.6'):
+ # no maxtaskperchild in 2.6
+ pool = multiprocessing.Pool(processes=max_processes)
+ else:
+ pool = multiprocessing.Pool(processes=max_processes, maxtasksperchild=4)
+
+ filecount = 0
+ for dirname, dirnames, filenames in os.walk(source_dir):
+ for filename in filenames:
+ if filename.endswith('.csv'):
+ # output files go to outputDir and are named using the last subdirectory from the dirname
+ if output_dir:
+ out_filename = filename.replace('csv', 'json')
+ out_filename = os.path.join(output_dir, '%s_%s' % (os.path.split(dirname)[-1], out_filename))
+
+ # if file does not exist or if overwrite is true, add file process to the pool
+ if not os.path.isfile(out_filename) or overwrite:
+ pool.apply_async(process_csv, args=(os.path.join(dirname, filename), out_filename))
+ filecount += 1
+ else:
+ logging.info("Skipping %s, %s exists and overwrite is false" % (filename, out_filename))
+ else:
+ # no outputdir so we just analyze the files
+ pool.apply_async(process_csv, args=(os.path.join(dirname, filename), None))
+ filecount += 1
+
+ try:
+ pool.close()
+ logging.info("Starting activities on %s CSV files" % filecount)
+ pool.join()
+ except KeyboardInterrupt:
+ logging.info("Aborting")
+ pool.terminate()
+
+ logging.info("Completed")
+
+
+##-------------------------------------------------------------------------
+
+if __name__ == "__main__":
+
+ max_cpu = multiprocessing.cpu_count()
+
+ from optparse import OptionParser
+ parser = OptionParser()
+ parser.add_option('-s', '--source', dest='source_dir', action='store',
+ help='Source directory to walk for CSV files')
+ parser.add_option('-o', '--output', dest='out_dir', action='store',
+ help='Output directory for JSON files')
+ parser.add_option('-O', '--overwrite', dest='overwrite', action='store_true',
+ help='Overwrite existing files in output directory')
+ parser.add_option('-p', '--processes', dest='max_processes', action='store', default=max_cpu, type='int',
+ help='Max number of processes to spawn')
+ parser.add_option('-a', '--analyze', dest='analyze', action='store_true',
+ help='Analyze CSV files for validity, no file output')
+ parser.add_option('-d', '--debug', dest='debug', action='store_true',
+ help='Enable debug messages')
+
+ (options, args) = parser.parse_args()
+
+ if not options.source_dir:
+ logging.error("Source directory required")
+ sys.exit(-1)
+
+ if not options.out_dir or options.analyze:
+ out_dir = None
+ elif not options.out_dir:
+ logging.error("Ouput directory or analysis option required")
+ sys.exit(-1)
+ else:
+ out_dir = options.out_dir
+
+ if options.max_processes > max_cpu:
+ logging.warn('Max Processes (%s) is greater than available Processors (%s)' % (options.max_processes, max_cpu))
+
+ if options.debug:
+ # enable debug level and multiprocessing debugging
+ logging.basicConfig(level=logging.DEBUG)
+ multiprocessing.log_to_stderr(logging.DEBUG)
+
+ process_files(options.source_dir, options.out_dir, options.max_processes, options.overwrite)
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh b/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
new file mode 100755
index 0000000..03fd641
--- /dev/null
+++ b/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
+[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
+
+# Autodetect JAVA_HOME if not defined
+if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+ . /usr/libexec/bigtop-detect-javahome
+elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+ . /usr/lib/bigtop-utils/bigtop-detect-javahome
+fi
+
+export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
+CP=/usr/metron/0.1BETA/lib/metron-data-management-0.1BETA.jar:/usr/metron/0.1BETA/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath`
+HADOOP_CLASSPATH=$(echo $CP )
+for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
+ if [ -f $jar ];then
+ LIBJARS="$jar,$LIBJARS"
+ fi
+done
+export HADOOP_CLASSPATH
+export METRON_VERSION=${project.version}
+export METRON_HOME=/usr/metron/$METRON_VERSION
+export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.nonbulk.flatfile.SimpleEnrichmentFlatFileLoader "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/scripts/prune_elasticsearch_indices.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/scripts/prune_elasticsearch_indices.sh b/metron-platform/metron-data-management/src/main/scripts/prune_elasticsearch_indices.sh
new file mode 100644
index 0000000..c3f1d05
--- /dev/null
+++ b/metron-platform/metron-data-management/src/main/scripts/prune_elasticsearch_indices.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}.jar org.apache.metron.dataloads.bulk.ElasticsearchDataPrunerRunner "$@"
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh b/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh
new file mode 100644
index 0000000..b8d9372
--- /dev/null
+++ b/metron-platform/metron-data-management/src/main/scripts/prune_hdfs_files.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+yarn jar /usr/metron/${project.version}/lib/${project.artifactId}-${project.version}.jar org.apache.metron.dataloads.bulk.HDFSDataPruner "$@"
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_load.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_load.sh b/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_load.sh
new file mode 100755
index 0000000..865d0ad
--- /dev/null
+++ b/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_load.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
+[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
+
+# Autodetect JAVA_HOME if not defined
+if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+ . /usr/libexec/bigtop-detect-javahome
+elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+ . /usr/lib/bigtop-utils/bigtop-detect-javahome
+fi
+
+export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
+HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
+for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
+ if [ -f $jar ];then
+ LIBJARS="$jar,$LIBJARS"
+ fi
+done
+export HADOOP_CLASSPATH
+export METRON_VERSION=${project.version}
+export METRON_HOME=/usr/metron/$METRON_VERSION
+export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.bulk.ThreatIntelBulkLoader -libjars ${LIBJARS} "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh b/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh
new file mode 100755
index 0000000..6156027
--- /dev/null
+++ b/metron-platform/metron-data-management/src/main/scripts/threatintel_bulk_prune.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
+[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
+
+# Autodetect JAVA_HOME if not defined
+if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+ . /usr/libexec/bigtop-detect-javahome
+elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+ . /usr/lib/bigtop-utils/bigtop-detect-javahome
+fi
+
+export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
+HADOOP_CLASSPATH=${HBASE_HOME}/lib/hbase-server.jar:`${HBASE_HOME}/bin/hbase classpath`
+for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
+ if [ -f $jar ];then
+ LIBJARS="$jar,$LIBJARS"
+ fi
+done
+export HADOOP_CLASSPATH
+export METRON_VERSION=${project.version}
+export METRON_HOME=/usr/metron/$METRON_VERSION
+export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.bulk.LeastRecentlyUsedPruner -libjars ${LIBJARS} "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh b/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh
new file mode 100755
index 0000000..23d09ba
--- /dev/null
+++ b/metron-platform/metron-data-management/src/main/scripts/threatintel_taxii_load.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
+[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
+
+# Autodetect JAVA_HOME if not defined
+if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+ . /usr/libexec/bigtop-detect-javahome
+elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+ . /usr/lib/bigtop-utils/bigtop-detect-javahome
+fi
+
+export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
+CP=/usr/metron/0.1BETA/lib/metron-data-management-0.1BETA.jar:/usr/metron/0.1BETA/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath`
+HADOOP_CLASSPATH=$(echo $CP )
+for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
+ if [ -f $jar ];then
+ LIBJARS="$jar,$LIBJARS"
+ fi
+done
+export HADOOP_CLASSPATH
+export METRON_VERSION=${project.version}
+export METRON_HOME=/usr/metron/$METRON_VERSION
+export DM_JAR=${project.artifactId}-$METRON_VERSION.jar
+hadoop jar $METRON_HOME/lib/$DM_JAR org.apache.metron.dataloads.nonbulk.taxii.TaxiiLoader "$@"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
index 6b54fec..989abfb 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
@@ -19,7 +19,7 @@ package org.apache.metron.elasticsearch.writer;
import backtype.storm.tuple.Tuple;
import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.common.interfaces.BulkMessageWriter;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml
index 756a0b4..87e5096 100644
--- a/metron-platform/metron-enrichment/pom.xml
+++ b/metron-platform/metron-enrichment/pom.xml
@@ -159,6 +159,7 @@
</systemProperties>
</configuration>
</plugin>
+
<!-- Normally, dependency report takes time, skip it -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
index f2e0113..22629a4 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
@@ -64,7 +64,7 @@ public class SimpleHBaseAdapter implements EnrichmentAdapter<CacheKey>,Serializa
public JSONObject enrich(CacheKey value) {
JSONObject enriched = new JSONObject();
List<String> enrichmentTypes = value.getConfig()
- .getFieldToEnrichmentTypeMap()
+ .getEnrichment().getFieldToTypeMap()
.get(EnrichmentUtils.toTopLevelField(value.getField()));
if(enrichmentTypes != null && value.getValue() != null) {
try {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
index dbdf6ec..ee5636b 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
@@ -54,7 +54,7 @@ public class ThreatIntelAdapter implements EnrichmentAdapter<CacheKey>,Serializa
@Override
public void logAccess(CacheKey value) {
- List<String> enrichmentTypes = value.getConfig().getFieldToThreatIntelTypeMap().get(value.getField());
+ List<String> enrichmentTypes = value.getConfig().getThreatIntel().getFieldToTypeMap().get(value.getField());
if(enrichmentTypes != null) {
for(String enrichmentType : enrichmentTypes) {
lookup.getAccessTracker().logAccess(new EnrichmentKey(enrichmentType, value.getValue()));
@@ -67,7 +67,7 @@ public class ThreatIntelAdapter implements EnrichmentAdapter<CacheKey>,Serializa
public JSONObject enrich(CacheKey value) {
JSONObject enriched = new JSONObject();
List<String> enrichmentTypes = value.getConfig()
- .getFieldToThreatIntelTypeMap()
+ .getThreatIntel().getFieldToTypeMap()
.get(EnrichmentUtils.toTopLevelField(value.getField()));
if(enrichmentTypes != null) {
int i = 0;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
index f3e742d..7cfa34d 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
@@ -24,7 +24,7 @@ import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import org.apache.metron.common.Constants;
import org.apache.metron.common.bolt.ConfiguredBolt;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.common.utils.ErrorUtils;
import org.apache.metron.common.utils.MessageUtils;
import org.apache.metron.common.interfaces.BulkMessageWriter;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/CacheKey.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/CacheKey.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/CacheKey.java
index 1338b44..4dbd33e 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/CacheKey.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/CacheKey.java
@@ -17,7 +17,7 @@
*/
package org.apache.metron.enrichment.bolt;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
public class CacheKey {
private String field;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
index 8ef44d0..48e09f8 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
@@ -18,7 +18,7 @@
package org.apache.metron.enrichment.bolt;
import backtype.storm.task.TopologyContext;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.common.utils.MessageUtils;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
@@ -48,6 +48,10 @@ public class EnrichmentJoinBolt extends JoinBolt<JSONObject> {
public Set<String> getStreamIds(JSONObject message) {
Set<String> streamIds = new HashSet<>();
String sourceType = MessageUtils.getSensorType(message);
+ if(sourceType == null) {
+ String errorMessage = "Unable to find source type for message: " + message;
+ throw new IllegalStateException(errorMessage);
+ }
Map<String, List<String>> fieldMap = getFieldMap(sourceType);
if(fieldMap != null) {
for (String enrichmentType : getFieldMap(sourceType).keySet()) {
@@ -83,8 +87,8 @@ public class EnrichmentJoinBolt extends JoinBolt<JSONObject> {
public Map<String, List<String>> getFieldMap(String sourceType) {
if(sourceType != null) {
SensorEnrichmentConfig config = configurations.getSensorEnrichmentConfig(sourceType);
- if (config != null) {
- return config.getEnrichmentFieldMap();
+ if (config != null && config.getEnrichment() != null) {
+ return config.getEnrichment().getFieldMap();
}
else {
LOG.error("Unable to retrieve a sensor enrichment config of " + sourceType);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
index 6b49edb..c367173 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
@@ -21,7 +21,7 @@ import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Tuple;
import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.enrichment.configuration.Enrichment;
import org.apache.metron.enrichment.utils.EnrichmentUtils;
import org.apache.metron.common.utils.MessageUtils;
@@ -127,7 +127,7 @@ public class EnrichmentSplitterBolt extends SplitBolt<JSONObject> {
if(sensorType != null) {
SensorEnrichmentConfig config = configurations.getSensorEnrichmentConfig(sensorType);
if (config != null) {
- return config.getEnrichmentFieldMap();
+ return config.getEnrichment().getFieldMap();
} else {
LOG.error("Unable to retrieve a sensor enrichment config of " + sensorType);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
index e5b8ca6..b79d6c7 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
@@ -31,7 +31,7 @@ import org.apache.metron.common.Constants;
import org.apache.metron.common.bolt.ConfiguredBolt;
import org.apache.metron.common.configuration.Configurations;
import org.apache.metron.enrichment.configuration.Enrichment;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.apache.metron.common.utils.ErrorUtils;
import org.json.simple.JSONObject;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
index 68e56ed..1964961 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
@@ -29,6 +29,8 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.Sets;
import org.apache.metron.common.bolt.ConfiguredBolt;
+import org.apache.metron.common.utils.ErrorUtils;
+import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
@@ -66,10 +68,12 @@ public abstract class JoinBolt<V> extends ConfiguredBolt {
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
super.prepare(map, topologyContext, outputCollector);
this.collector = outputCollector;
- if (this.maxCacheSize == null)
+ if (this.maxCacheSize == null) {
throw new IllegalStateException("maxCacheSize must be specified");
- if (this.maxTimeRetain == null)
+ }
+ if (this.maxTimeRetain == null) {
throw new IllegalStateException("maxTimeRetain must be specified");
+ }
loader = new CacheLoader<String, Map<String, V>>() {
public Map<String, V> load(String key) throws Exception {
return new HashMap<>();
@@ -96,13 +100,18 @@ public abstract class JoinBolt<V> extends ConfiguredBolt {
streamMessageMap.put(streamId, message);
Set<String> streamIds = getStreamIds(message);
Set<String> streamMessageKeys = streamMessageMap.keySet();
- if (streamMessageKeys.size() == streamIds.size() && Sets.symmetricDifference
- (streamMessageKeys, streamIds)
- .isEmpty()) {
- collector.emit("message", tuple, new Values(key, joinMessages
- (streamMessageMap)));
- collector.ack(tuple);
+ if ( streamMessageKeys.size() == streamIds.size()
+ && Sets.symmetricDifference(streamMessageKeys, streamIds)
+ .isEmpty()
+ ) {
+ collector.emit( "message"
+ , tuple
+ , new Values( key
+ , joinMessages(streamMessageMap)
+ )
+ );
cache.invalidate(key);
+ collector.ack(tuple);
} else {
cache.put(key, streamMessageMap);
if(LOG.isDebugEnabled()) {
@@ -111,15 +120,19 @@ public abstract class JoinBolt<V> extends ConfiguredBolt {
);
}
}
- } catch (ExecutionException e) {
+ } catch (Exception e) {
+ LOG.error("[Metron] Unable to join messages: " + message, e);
+ JSONObject error = ErrorUtils.generateErrorMessage("Joining problem: " + message, e);
+ collector.ack(tuple);
+ collector.emit("error", new Values(error));
collector.reportError(e);
- LOG.error(e.getMessage(), e);
}
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declareStream("message", new Fields("key", "message"));
+ declarer.declareStream("error", new Fields("message"));
}
public abstract void prepare(Map map, TopologyContext topologyContext);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
index a2b0e78..c08bd0d 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
@@ -17,7 +17,12 @@
*/
package org.apache.metron.enrichment.bolt;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import com.google.common.base.Joiner;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig;
+import org.apache.metron.common.utils.MessageUtils;
+import org.apache.metron.threatintel.triage.ThreatTriageProcessor;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,7 +43,7 @@ public class ThreatIntelJoinBolt extends EnrichmentJoinBolt {
public Map<String, List<String>> getFieldMap(String sourceType) {
SensorEnrichmentConfig config = configurations.getSensorEnrichmentConfig(sourceType);
if(config != null) {
- return config.getThreatIntelFieldMap();
+ return config.getThreatIntel().getFieldMap();
}
else {
LOG.error("Unable to retrieve sensor config: " + sourceType);
@@ -49,12 +54,54 @@ public class ThreatIntelJoinBolt extends EnrichmentJoinBolt {
@Override
public JSONObject joinMessages(Map<String, JSONObject> streamMessageMap) {
JSONObject ret = super.joinMessages(streamMessageMap);
- for(Object key : ret.keySet()) {
- if(key.toString().startsWith("threatintels") && !key.toString().endsWith(".ts")) {
- ret.put("is_alert" , "true");
- break;
+ boolean isAlert = ret.containsKey("is_alert");
+ if(!isAlert) {
+ for (Object key : ret.keySet()) {
+ if (key.toString().startsWith("threatintels") && !key.toString().endsWith(".ts")) {
+ isAlert = true;
+ break;
+ }
}
}
+ if(isAlert) {
+ ret.put("is_alert" , "true");
+ String sourceType = MessageUtils.getSensorType(ret);
+ SensorEnrichmentConfig config = configurations.getSensorEnrichmentConfig(sourceType);
+ ThreatTriageConfig triageConfig = null;
+ if(config != null) {
+ triageConfig = config.getThreatIntel().getTriageConfig();
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(sourceType + ": Found sensor enrichment config.");
+ }
+ }
+ else {
+ LOG.debug(sourceType + ": Unable to find threat config.");
+ }
+ if(triageConfig != null) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(sourceType + ": Found threat triage config: " + triageConfig);
+ }
+
+ if(LOG.isDebugEnabled() && (triageConfig.getRiskLevelRules() == null || triageConfig.getRiskLevelRules().isEmpty())) {
+ LOG.debug(sourceType + ": Empty rules!");
+ }
+
+ ThreatTriageProcessor threatTriageProcessor = new ThreatTriageProcessor(triageConfig);
+ Double triageLevel = threatTriageProcessor.apply(ret);
+ if(LOG.isDebugEnabled()) {
+ String rules = Joiner.on('\n').join(triageConfig.getRiskLevelRules().entrySet());
+ LOG.debug("Marked " + sourceType + " as triage level " + triageLevel + " with rules " + rules);
+ }
+ if(triageLevel != null && triageLevel > 0) {
+ ret.put("threat.triage.level", triageLevel);
+ }
+ }
+ else {
+ LOG.debug(sourceType + ": Unable to find threat triage config!");
+ }
+
+ }
+
return ret;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
index 1429b2c..3cd1780 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
@@ -17,7 +17,7 @@
*/
package org.apache.metron.enrichment.bolt;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.enrichment.utils.ThreatIntelUtils;
import java.util.HashMap;
@@ -35,7 +35,7 @@ public class ThreatIntelSplitterBolt extends EnrichmentSplitterBolt {
if (sensorType != null) {
SensorEnrichmentConfig config = configurations.getSensorEnrichmentConfig(sensorType);
if (config != null) {
- return config.getThreatIntelFieldMap();
+ return config.getThreatIntel().getFieldMap();
} else {
LOG.error("Unable to retrieve sensor config: " + sensorType);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
new file mode 100644
index 0000000..bb4d387
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.threatintel.triage;
+
+import com.google.common.base.Function;
+import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig;
+import org.apache.metron.common.query.MapVariableResolver;
+import org.apache.metron.common.query.PredicateProcessor;
+import org.apache.metron.common.query.VariableResolver;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class ThreatTriageProcessor implements Function<Map, Double> {
+ private ThreatTriageConfig config;
+ public ThreatTriageProcessor(ThreatTriageConfig config) {
+ this.config = config;
+ }
+
+ @Nullable
+ @Override
+ public Double apply(@Nullable Map input) {
+ List<Number> scores = new ArrayList<>();
+ PredicateProcessor predicateProcessor = new PredicateProcessor();
+ VariableResolver resolver = new MapVariableResolver(input);
+ for(Map.Entry<String, Number> kv : config.getRiskLevelRules().entrySet()) {
+ if(predicateProcessor.parse(kv.getKey(), resolver)) {
+ scores.add(kv.getValue());
+ }
+ }
+ return config.getAggregator().aggregate(scores, config.getAggregationConfig());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
index 1c79f12..e2c6309 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
@@ -19,7 +19,7 @@ package org.apache.metron.enrichment.adapters.simplehbase;
import org.adrianwalker.multilinestring.Multiline;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.enrichment.bolt.CacheKey;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
@@ -52,26 +52,28 @@ public class SimpleHBaseAdapterTest {
}};
/**
- * {
- * "10.0.2.3.orientation":"north"
- * }
+ {
+ "10.0.2.3.orientation":"north"
+ }
*/
@Multiline
private String expectedMessageString;
/**
- * {
- * "index": "bro",
- * "batchSize": 5,
- * "enrichmentFieldMap": {
- * "geo": ["ip_dst_addr", "ip_src_addr"],
- * "host": ["host"]
- * },
- * "fieldToEnrichmentTypeMap": {
- * "ip_dst_addr" : [ "10.0.2.3" ],
- * "ip_src_addr" : [ "10.3.30.120" ]
- * }
- * }
+ {
+ "index": "bro",
+ "batchSize": 5,
+ "enrichment": {
+ "fieldMap": {
+ "geo": ["ip_dst_addr", "ip_src_addr"],
+ "host": ["host"]
+ },
+ "fieldToTypeMap": {
+ "ip_dst_addr" : [ "10.0.2.3" ],
+ "ip_src_addr" : [ "10.3.30.120" ]
+ }
+ }
+ }
*/
@Multiline
private String sourceConfigStr;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
index 62c8b43..2afeb5b 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
@@ -20,7 +20,7 @@ package org.apache.metron.enrichment.adapters.threatintel;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.enrichment.bolt.CacheKey;
import org.apache.metron.hbase.TableProvider;
import org.apache.metron.enrichment.converter.EnrichmentKey;
@@ -62,29 +62,33 @@ public class ThreatIntelAdapterTest {
private EnrichmentLookup lookup;
/**
- * {
- * "10.0.2.3":"alert"
- * }
+ {
+ "10.0.2.3":"alert"
+ }
*/
@Multiline
private String expectedMessageString;
/**
- * {
- * "index": "bro",
- * "batchSize": 5,
- * "enrichmentFieldMap": {
- * "geo": ["ip_dst_addr", "ip_src_addr"],
- * "host": ["host"]
- * },
- * "threatIntelFieldMap": {
- * "hbaseThreatIntel": ["ip_dst_addr", "ip_src_addr"]
- * },
- * "fieldToThreatIntelTypeMap": {
- * "ip_dst_addr" : [ "10.0.2.3" ],
- * "ip_src_addr" : [ "malicious_ip" ]
- * }
- * }
+ {
+ "index": "bro",
+ "batchSize": 5,
+ "enrichment": {
+ "fieldMap": {
+ "geo": ["ip_dst_addr", "ip_src_addr"],
+ "host": ["host"]
+ }
+ },
+ "threatIntel" : {
+ "fieldMap": {
+ "hbaseThreatIntel": ["ip_dst_addr", "ip_src_addr"]
+ },
+ "fieldToTypeMap": {
+ "ip_dst_addr" : [ "10.0.2.3" ],
+ "ip_src_addr" : [ "malicious_ip" ]
+ }
+ }
+ }
*/
@Multiline
private static String sourceConfigStr;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
index d5a90fb..5a09f79 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
@@ -22,9 +22,9 @@ import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.TestConstants;
import org.apache.metron.test.bolt.BaseEnrichmentBoltTest;
import org.apache.metron.enrichment.configuration.Enrichment;
-import org.apache.metron.common.configuration.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-import org.apache.metron.common.cli.ConfigurationsUtils;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.hamcrest.Description;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/deed21e6/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
index 306c3e1..d3a5414 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
@@ -17,8 +17,12 @@
*/
package org.apache.metron.enrichment.bolt;
+import com.fasterxml.jackson.databind.JsonMappingException;
import junit.framework.Assert;
import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.common.configuration.enrichment.threatintel.ThreatTriageConfig;
+import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.test.bolt.BaseEnrichmentBoltTest;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
@@ -78,12 +82,37 @@ public class ThreatIntelJoinBoltTest extends BaseEnrichmentBoltTest {
alertMessage = (JSONObject) parser.parse(alertMessageString);
}
- @Test
- public void test() throws IOException {
+ /**
+ {
+ "riskLevelRules" : {
+ "enrichedField1 == 'enrichedValue1'" : 10
+ }
+ ,"aggregator" : "MAX"
+ }
+ */
+ @Multiline
+ private static String threatTriageConfigStr;
+
+ public void test(String threatTriageConfig, boolean badConfig) throws IOException {
ThreatIntelJoinBolt threatIntelJoinBolt = new ThreatIntelJoinBolt("zookeeperUrl");
threatIntelJoinBolt.setCuratorFramework(client);
threatIntelJoinBolt.setTreeCache(cache);
- threatIntelJoinBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(sampleSensorEnrichmentConfigPath));
+ SensorEnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(new FileInputStream(sampleSensorEnrichmentConfigPath), SensorEnrichmentConfig.class);
+ boolean withThreatTriage = threatTriageConfig != null;
+ if(withThreatTriage) {
+ try {
+ enrichmentConfig.getThreatIntel().setTriageConfig(JSONUtils.INSTANCE.load(threatTriageConfig, ThreatTriageConfig.class));
+ if(badConfig) {
+ Assert.fail(threatTriageConfig + "\nThis should not parse!");
+ }
+ }
+ catch(JsonMappingException pe) {
+ if(!badConfig) {
+ throw pe;
+ }
+ }
+ }
+ threatIntelJoinBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, enrichmentConfig);
threatIntelJoinBolt.withMaxCacheSize(100);
threatIntelJoinBolt.withMaxTimeRetain(10000);
threatIntelJoinBolt.prepare(new HashMap<>(), topologyContext, outputCollector);
@@ -101,5 +130,35 @@ public class ThreatIntelJoinBoltTest extends BaseEnrichmentBoltTest {
streamMessageMap.put("message", alertMessage);
joinedMessage = threatIntelJoinBolt.joinMessages(streamMessageMap);
Assert.assertTrue(joinedMessage.containsKey("is_alert") && "true".equals(joinedMessage.get("is_alert")));
+ if(withThreatTriage && !badConfig) {
+ Assert.assertTrue(joinedMessage.containsKey("threat.triage.level") && Math.abs(10d - (Double) joinedMessage.get("threat.triage.level")) < 1e-10);
+ }
+ else {
+ Assert.assertFalse(joinedMessage.containsKey("threat.triage.level"));
+ }
+ }
+ /**
+ {
+ "riskLevelRules" : {
+ "enrichedField1 == 'enrichedValue1" : 10
+ }
+ ,"aggregator" : "MAX"
+ }
+ */
+ @Multiline
+ private static String badRuleThreatTriageConfigStr;
+
+
+ @Test
+ public void testWithTriage() throws IOException {
+ test(threatTriageConfigStr, false);
+ }
+ @Test
+ public void testWithBadTriageRule() throws IOException {
+ test(badRuleThreatTriageConfigStr, true);
+ }
+ @Test
+ public void testWithoutTriage() throws IOException {
+ test(null, false);
}
}