You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by si...@apache.org on 2016/01/14 18:04:15 UTC
[80/85] incubator-metron git commit: Move all com/apache folders to
org/apache
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/alerts/adapters/ThreatAlertsAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/alerts/adapters/ThreatAlertsAdapter.java b/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/alerts/adapters/ThreatAlertsAdapter.java
deleted file mode 100644
index 48d773d..0000000
--- a/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/alerts/adapters/ThreatAlertsAdapter.java
+++ /dev/null
@@ -1,311 +0,0 @@
-package com.apache.metron.alerts.adapters;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.validator.routines.InetAddressValidator;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.apache.metron.alerts.interfaces.AlertsAdapter;
-
-@SuppressWarnings("serial")
-public class ThreatAlertsAdapter implements AlertsAdapter, Serializable {
-
- String enrichment_tag;
-
- HTableInterface blacklist_table;
- HTableInterface whitelist_table;
- InetAddressValidator ipvalidator = new InetAddressValidator();
- String _whitelist_table_name;
- String _blacklist_table_name;
- String _quorum;
- String _port;
- String _topologyname;
- Configuration conf = null;
-
- Cache<String, String> cache;
- String _topology_name;
-
- Set<String> loaded_whitelist = new HashSet<String>();
- Set<String> loaded_blacklist = new HashSet<String>();
-
- protected static final Logger LOG = LoggerFactory
- .getLogger(ThreatAlertsAdapter.class);
-
- public ThreatAlertsAdapter(Map<String, String> config) {
- try {
-
- if (!config.containsKey("whitelist_table_name"))
- throw new Exception("Whitelist table name is missing");
-
- _whitelist_table_name = config.get("whitelist_table_name");
-
- if (!config.containsKey("blacklist_table_name"))
- throw new Exception("Blacklist table name is missing");
-
- _blacklist_table_name = config.get("blacklist_table_name");
-
- if (!config.containsKey("quorum"))
- throw new Exception("Quorum name is missing");
-
- _quorum = config.get("quorum");
-
- if (!config.containsKey("port"))
- throw new Exception("port name is missing");
-
- _port = config.get("port");
-
- if (!config.containsKey("_MAX_CACHE_SIZE_OBJECTS_NUM"))
- throw new Exception("_MAX_CACHE_SIZE_OBJECTS_NUM name is missing");
-
- int _MAX_CACHE_SIZE_OBJECTS_NUM = Integer.parseInt(config
- .get("_MAX_CACHE_SIZE_OBJECTS_NUM"));
-
- if (!config.containsKey("_MAX_TIME_RETAIN_MINUTES"))
- throw new Exception("_MAX_TIME_RETAIN_MINUTES name is missing");
-
- int _MAX_TIME_RETAIN_MINUTES = Integer.parseInt(config
- .get("_MAX_TIME_RETAIN_MINUTES"));
-
- cache = CacheBuilder.newBuilder().maximumSize(_MAX_CACHE_SIZE_OBJECTS_NUM)
- .expireAfterWrite(_MAX_TIME_RETAIN_MINUTES, TimeUnit.MINUTES)
- .build();
-
- enrichment_tag = config.get("enrichment_tag");
-
- } catch (Exception e) {
- System.out.println("Could not initialize alerts adapter");
- e.printStackTrace();
- System.exit(0);
- }
- }
-
- @SuppressWarnings("resource")
- @Override
- public boolean initialize() {
-
- conf = HBaseConfiguration.create();
- // conf.set("hbase.zookeeper.quorum", _quorum);
- // conf.set("hbase.zookeeper.property.clientPort", _port);
-
- LOG.trace("[Metron] Connecting to hbase with conf:" + conf);
- LOG.trace("[Metron] Whitelist table name: " + _whitelist_table_name);
- LOG.trace("[Metron] Whitelist table name: " + _blacklist_table_name);
- LOG.trace("[Metron] ZK Client/port: "
- + conf.get("hbase.zookeeper.quorum") + " -> "
- + conf.get("hbase.zookeeper.property.clientPort"));
-
- try {
-
- LOG.trace("[Metron] Attempting to connect to hbase");
-
- HConnection connection = HConnectionManager.createConnection(conf);
-
- LOG.trace("[Metron] CONNECTED TO HBASE");
-
- HBaseAdmin hba = new HBaseAdmin(conf);
-
- if (!hba.tableExists(_whitelist_table_name))
- throw new Exception("Whitelist table doesn't exist");
-
- if (!hba.tableExists(_blacklist_table_name))
- throw new Exception("Blacklist table doesn't exist");
-
- whitelist_table = new HTable(conf, _whitelist_table_name);
-
- LOG.trace("[Metron] CONNECTED TO TABLE: " + _whitelist_table_name);
- blacklist_table = new HTable(conf, _blacklist_table_name);
- LOG.trace("[Metron] CONNECTED TO TABLE: " + _blacklist_table_name);
-
- if (connection == null || whitelist_table == null
- || blacklist_table == null)
- throw new Exception("Unable to initialize hbase connection");
-
- Scan scan = new Scan();
-
- ResultScanner rs = whitelist_table.getScanner(scan);
- try {
- for (Result r = rs.next(); r != null; r = rs.next()) {
- loaded_whitelist.add(Bytes.toString(r.getRow()));
- }
- } catch (Exception e) {
- LOG.trace("[Metron] COULD NOT READ FROM HBASE");
- e.printStackTrace();
- } finally {
- rs.close(); // always close the ResultScanner!
- hba.close();
- }
- whitelist_table.close();
-
- LOG.trace("[Metron] READ IN WHITELIST: " + loaded_whitelist.size());
-
- scan = new Scan();
-
- rs = blacklist_table.getScanner(scan);
- try {
- for (Result r = rs.next(); r != null; r = rs.next()) {
- loaded_blacklist.add(Bytes.toString(r.getRow()));
- }
- } catch (Exception e) {
- LOG.trace("[Metron] COULD NOT READ FROM HBASE");
- e.printStackTrace();
- } finally {
- rs.close(); // always close the ResultScanner!
- hba.close();
- }
- blacklist_table.close();
-
- LOG.trace("[Metron] READ IN WHITELIST: " + loaded_whitelist.size());
-
- rs.close(); // always close the ResultScanner!
- hba.close();
-
- return true;
- } catch (Exception e) {
-
- e.printStackTrace();
- }
-
- return false;
-
- }
-
- @Override
- public boolean refresh() throws Exception {
- return true;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public Map<String, JSONObject> alert(JSONObject raw_message) {
-
- System.out.println("LOOKING FOR ENRICHMENT TAG: " + enrichment_tag);
-
- Map<String, JSONObject> alerts = new HashMap<String, JSONObject>();
- JSONObject content = (JSONObject) raw_message.get("message");
-
- JSONObject enrichment = null;
-
- if (raw_message.containsKey("enrichment"))
- enrichment = (JSONObject) raw_message.get("enrichment");
- else
- return null;
-
- if (enrichment.containsKey(enrichment_tag)) {
-
- System.out.println("FOUND TAG: " + enrichment_tag);
-
- JSONObject threat = (JSONObject) enrichment.get(enrichment_tag);
-
- int cnt = 0;
- Object enriched_key = null;
-
- for (Object key : threat.keySet()) {
- JSONObject tmp = (JSONObject) threat.get(key);
- cnt = cnt + tmp.size();
- if (tmp.size() > 0)
- enriched_key = key;
- }
-
- if (cnt == 0) {
- System.out.println("TAG HAS NO ELEMENTS");
- return null;
- }
-
- JSONObject alert = new JSONObject();
-
- String source = "unknown";
- String dest = "unknown";
- String host = "unknown";
-
- if (content.containsKey("ip_src_addr")) {
- source = content.get("ip_src_addr").toString();
-
- if (RangeChecker.checkRange(loaded_whitelist, source))
- host = source;
- }
-
- if (content.containsKey("ip_dst_addr")) {
- dest = content.get("ip_dst_addr").toString();
-
- if (RangeChecker.checkRange(loaded_whitelist, dest))
- host = dest;
- }
-
- JSONObject threatQualifier = (JSONObject) threat.get(enriched_key);
-
- alert.put("designated_host", host);
- String description =
-
- new StringBuilder()
- .append("Threat Intelligence match for ")
- .append(content.get(enriched_key).toString())
- .append(" from source: ")
- .append(threatQualifier.keySet().iterator().next().toString())
- .toString();
- alert.put("description", description);
- alert.put("priority", "MED");
-
- String alert_id = generateAlertId(source, dest, 0);
-
- alert.put("alert_id", alert_id);
- alerts.put(alert_id, alert);
-
- alert.put("enrichment", enrichment);
-
- return alerts;
- } else {
- System.out.println("DID NOT FIND TAG: " + enrichment_tag);
- return null;
- }
-
- }
-
- @Override
- public boolean containsAlertId(String alert) {
- // TODO Auto-generated method stub
- return false;
- }
-
- protected String generateAlertId(String source_ip, String dst_ip,
- int alert_type) {
-
- String key = makeKey(source_ip, dst_ip, alert_type);
-
- if (cache.getIfPresent(key) != null)
- return cache.getIfPresent(key);
-
- String new_UUID = System.currentTimeMillis() + "-" + UUID.randomUUID();
-
- cache.put(key, new_UUID);
- key = makeKey(dst_ip, source_ip, alert_type);
- cache.put(key, new_UUID);
-
- return new_UUID;
-
- }
-
- private String makeKey(String ip1, String ip2, int alert_type) {
- return (ip1 + "-" + ip2 + "-" + alert_type);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/AbstractTaggerBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/AbstractTaggerBolt.java b/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/AbstractTaggerBolt.java
deleted file mode 100644
index 72eaacd..0000000
--- a/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/AbstractTaggerBolt.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.apache.metron.tagging;
-
-import java.io.IOException;
-import java.util.Map;
-
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.topology.base.BaseRichBolt;
-import backtype.storm.tuple.Fields;
-
-import com.codahale.metrics.Counter;
-import com.apache.metron.alerts.interfaces.TaggerAdapter;
-import com.apache.metron.metrics.MetricReporter;
-
-@SuppressWarnings("rawtypes")
-public abstract class AbstractTaggerBolt extends BaseRichBolt {
- /**
- *
- */
- private static final long serialVersionUID = -6710596708304282838L;
-
- protected static final Logger LOG = LoggerFactory
- .getLogger(AbstractTaggerBolt.class);
-
- protected OutputCollector _collector;
- protected TaggerAdapter _adapter;
-
- protected String OutputFieldName;
- protected JSONObject _identifier;
- protected MetricReporter _reporter;
-
- protected Counter ackCounter, emitCounter, failCounter;
-
- protected void registerCounters() {
-
- String ackString = _adapter.getClass().getSimpleName() + ".ack";
-
- String emitString = _adapter.getClass().getSimpleName() + ".emit";
-
- String failString = _adapter.getClass().getSimpleName() + ".fail";
-
- ackCounter = _reporter.registerCounter(ackString);
- emitCounter = _reporter.registerCounter(emitString);
- failCounter = _reporter.registerCounter(failString);
-
- }
-
- public final void prepare(Map conf, TopologyContext topologyContext,
- OutputCollector collector) {
- _collector = collector;
-
- if (this._adapter == null)
- throw new IllegalStateException("Tagging must be specified");
- if(this._identifier == null)
- throw new IllegalStateException("Identifier must be specified");
- try {
- doPrepare(conf, topologyContext, collector);
- } catch (IOException e) {
- LOG.error("Counld not initialize...");
- e.printStackTrace();
- }
- }
-
- public void declareOutputFields(OutputFieldsDeclarer declearer) {
- declearer.declare(new Fields(this.OutputFieldName));
- }
-
- abstract void doPrepare(Map conf, TopologyContext topologyContext,
- OutputCollector collector) throws IOException;
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/TelemetryTaggerBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/TelemetryTaggerBolt.java b/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/TelemetryTaggerBolt.java
deleted file mode 100644
index 69f87b1..0000000
--- a/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/TelemetryTaggerBolt.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.tagging;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.commons.configuration.Configuration;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Fields;
-import backtype.storm.tuple.Tuple;
-import backtype.storm.tuple.Values;
-
-import com.apache.metron.alerts.interfaces.TaggerAdapter;
-import com.apache.metron.json.serialization.JSONEncoderHelper;
-import com.apache.metron.metrics.MetricReporter;
-
-@SuppressWarnings("rawtypes")
-public class TelemetryTaggerBolt extends AbstractTaggerBolt {
-
- /**
- * Use an adapter to tag existing telemetry messages with alerts. The list
- * of available tagger adapters is located under
- * com.apache.metron.tagging.adapters. At the time of the release the following
- * adapters are available:
- *
- * <p>
- * <ul>
- * <li>RegexTagger = read a list or regular expressions and tag a message if
- * they exist in a message
- * <li>StaticAllTagger = tag each message with a static alert
- * <ul>
- * <p>
- */
- private static final long serialVersionUID = -2647123143398352020L;
- private Properties metricProperties;
- private JSONObject metricConfiguration;
-
- /**
- *
- * @param tagger
- * - tagger adapter for generating alert tags
- * @return instance of bolt
- */
- public TelemetryTaggerBolt withMessageTagger(TaggerAdapter tagger) {
- _adapter = tagger;
- return this;
- }
-
- /**
- *
- * @param OutputFieldName
- * - output name of the tuple coming out of this bolt
- * @return - instance of this bolt
- */
- public TelemetryTaggerBolt withOutputFieldName(String OutputFieldName) {
- this.OutputFieldName = OutputFieldName;
- return this;
- }
-
- /**
- *
- * @param metricProperties
- * - metric output to graphite
- * @return - instance of this bolt
- */
- public TelemetryTaggerBolt withMetricProperties(Properties metricProperties) {
- this.metricProperties = metricProperties;
- return this;
- }
-
- /**
- *
- * @param identifier
- * - the identifier tag for tagging telemetry messages with
- * alerts out of this bolt
- * @return - instance of this bolt
- */
-
- public TelemetryTaggerBolt withIdentifier(JSONObject identifier) {
- this._identifier = identifier;
- return this;
- }
-
- /**
- * @param config
- * A class for generating custom metrics into graphite
- * @return Instance of this class
- */
-
- public TelemetryTaggerBolt withMetricConfiguration(Configuration config) {
- this.metricConfiguration = JSONEncoderHelper.getJSON(config
- .subset("com.apache.metron.metrics"));
- return this;
- }
-
- @Override
- void doPrepare(Map conf, TopologyContext topologyContext,
- OutputCollector collector) throws IOException {
-
- LOG.info("[Metron] Preparing TelemetryParser Bolt...");
-
- try {
- _reporter = new MetricReporter();
- _reporter.initialize(metricProperties, TelemetryTaggerBolt.class);
- LOG.info("[Metron] Initialized metrics");
- } catch (Exception e) {
- LOG.info("[Metron] Could not initialize metrics");
- }
- }
-
- @SuppressWarnings("unchecked")
- public void execute(Tuple tuple) {
-
- LOG.trace("[Metron] Starting to process message for alerts");
- JSONObject original_message = null;
-
- try {
-
- original_message = (JSONObject) tuple.getValue(0);
-
- if (original_message == null || original_message.isEmpty())
- throw new Exception("Could not parse message from byte stream");
-
- LOG.trace("[Metron] Received tuple: " + original_message);
-
- JSONObject alerts_tag = new JSONObject();
- JSONArray alerts_list = _adapter.tag(original_message);
-
- LOG.trace("[Metron] Tagged message: " + alerts_list);
-
- if (alerts_list.size() != 0) {
- if (original_message.containsKey("alerts")) {
- JSONObject tag = (JSONObject) original_message
- .get("alerts");
- JSONArray already_triggered = (JSONArray) tag
- .get("triggered");
- alerts_list.addAll(already_triggered);
- LOG.trace("[Metron] Created a new string of alerts");
- }
-
- alerts_tag.put("identifier", _identifier);
- alerts_tag.put("triggered", alerts_list);
- original_message.put("alerts", alerts_tag);
-
- LOG.debug("[Metron] Detected alerts: " + alerts_tag);
- }
- else
- {
- LOG.debug("[Metron] The following messages did not contain alerts: " + original_message);
- }
-
- _collector.ack(tuple);
- _collector.emit(new Values(original_message));
-
- /*if (metricConfiguration != null) {
- emitCounter.inc();
- ackCounter.inc();
- }*/
-
- } catch (Exception e) {
- e.printStackTrace();
- LOG.error("Failed to tag message :" + original_message);
- e.printStackTrace();
- _collector.fail(tuple);
-
- /*
- if (metricConfiguration != null) {
- failCounter.inc();
- }*/
- }
- }
-
- public void declareOutputFields(OutputFieldsDeclarer declearer) {
- declearer.declare(new Fields(this.OutputFieldName));
-
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/AbstractTaggerAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/AbstractTaggerAdapter.java b/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/AbstractTaggerAdapter.java
deleted file mode 100644
index 8d56e6f..0000000
--- a/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/AbstractTaggerAdapter.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.tagging.adapters;
-
-import java.io.Serializable;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.apache.metron.alerts.interfaces.TaggerAdapter;
-
-@SuppressWarnings("serial")
-public abstract class AbstractTaggerAdapter implements TaggerAdapter, Serializable{
-
- protected static final Logger _LOG = LoggerFactory
- .getLogger(AbstractTaggerAdapter.class);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/RegexTagger.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/RegexTagger.java b/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/RegexTagger.java
deleted file mode 100644
index 2854ae6..0000000
--- a/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/RegexTagger.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.tagging.adapters;
-
-import java.util.Map;
-
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-
-public class RegexTagger extends AbstractTaggerAdapter{
-
- /**
- * Reads a regex rules file and tags a message with alerts if any rule from that file
- * matches anything in the telemetry message
- */
- private static final long serialVersionUID = -6091495636459799411L;
- Map <String, JSONObject> _rules;
-
- /**
- *
- * @param rules rules read from a properties XML file
- */
- public RegexTagger(Map<String, JSONObject> rules)
- {
- _rules = rules;
- }
-
- /**
- * @param raw_message telemetry message to be tagged
- */
- @SuppressWarnings("unchecked")
- public JSONArray tag(JSONObject raw_message) {
-
- JSONArray ja = new JSONArray();
- String message_as_string = raw_message.toString();
-
- for(String rule : _rules.keySet())
- {
- if (message_as_string.matches(rule))
- {
- ja.add(_rules.get(rule));
- }
- }
-
- return ja;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/StaticAllTagger.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/StaticAllTagger.java b/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/StaticAllTagger.java
deleted file mode 100644
index 3962620..0000000
--- a/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/StaticAllTagger.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.tagging.adapters;
-
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-
-public class StaticAllTagger extends AbstractTaggerAdapter {
-
- /**
- * Attaches a static alerts tag to JSON telemetry messages
- */
- private static final long serialVersionUID = 7759427661169094065L;
- private JSONObject _static_tag_message;
- JSONArray ja = new JSONArray();
-
- /**
- *
- * @param static_tag_message
- * static alerts tag to attach to the message as a JSON
- */
- @SuppressWarnings("unchecked")
- public StaticAllTagger(JSONObject static_tag_message) {
- _static_tag_message = static_tag_message;
- ja.add(_static_tag_message);
- }
-
- /**
- * @param raw_message
- * message to tag
- */
- public JSONArray tag(JSONObject raw_message) {
-
- return ja;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/TaggerAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/TaggerAdapter.java b/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/TaggerAdapter.java
deleted file mode 100644
index b3ba635..0000000
--- a/metron-streaming/Metron-Alerts/src/main/java/com/apache/metron/tagging/adapters/TaggerAdapter.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package com.apache.metron.tagging.adapters;
-
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-
-public interface TaggerAdapter {
-
- JSONArray tag(JSONObject raw_message);
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/AbstractAlertBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/AbstractAlertBolt.java b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/AbstractAlertBolt.java
new file mode 100644
index 0000000..2d172d1
--- /dev/null
+++ b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/AbstractAlertBolt.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.apache.metron.alerts;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.tuple.Fields;
+
+import com.codahale.metrics.Counter;
+import com.google.common.cache.Cache;
+import com.apache.metron.alerts.interfaces.AlertsAdapter;
+import com.apache.metron.metrics.MetricReporter;
+
+@SuppressWarnings("rawtypes")
+public abstract class AbstractAlertBolt extends BaseRichBolt {
+ /**
+ *
+ */
+ private static final long serialVersionUID = -6710596708304282838L;
+
+ transient Cache<String, String> cache;
+
+ protected static final Logger LOG = LoggerFactory
+ .getLogger(AbstractAlertBolt.class);
+
+ protected OutputCollector _collector;
+ protected AlertsAdapter _adapter;
+
+ protected String OutputFieldName;
+ protected JSONObject _identifier;
+ protected MetricReporter _reporter;
+
+ protected int _MAX_CACHE_SIZE_OBJECTS_NUM = -1;
+ protected int _MAX_TIME_RETAIN_MINUTES = -1;
+
+ protected Counter ackCounter, emitCounter, failCounter;
+
+ protected void registerCounters() {
+
+ String ackString = _adapter.getClass().getSimpleName() + ".ack";
+
+ String emitString = _adapter.getClass().getSimpleName() + ".emit";
+
+ String failString = _adapter.getClass().getSimpleName() + ".fail";
+
+ ackCounter = _reporter.registerCounter(ackString);
+ emitCounter = _reporter.registerCounter(emitString);
+ failCounter = _reporter.registerCounter(failString);
+
+ }
+
+ public final void prepare(Map conf, TopologyContext topologyContext,
+ OutputCollector collector) {
+ _collector = collector;
+
+ if (this._adapter == null)
+ throw new IllegalStateException("Alerts adapter must be specified");
+ if (this._identifier == null)
+ throw new IllegalStateException("Identifier must be specified");
+
+ if (this._MAX_CACHE_SIZE_OBJECTS_NUM == -1)
+ throw new IllegalStateException("MAX_CACHE_SIZE_OBJECTS_NUM must be specified");
+ if (this._MAX_TIME_RETAIN_MINUTES == -1)
+ throw new IllegalStateException("MAX_TIME_RETAIN_MINUTES must be specified");
+
+ try {
+ doPrepare(conf, topologyContext, collector);
+ } catch (IOException e) {
+ LOG.error("Counld not initialize...");
+ e.printStackTrace();
+ }
+
+ boolean success = _adapter.initialize();
+
+ try {
+ if (!success)
+
+ throw new Exception("Could not initialize adapter");
+ } catch (Exception e) {
+
+ e.printStackTrace();
+ }
+ }
+
+ public void declareOutputFields(OutputFieldsDeclarer declearer) {
+ declearer.declareStream("message", new Fields("key", "message"));
+ declearer.declareStream("alert", new Fields( "message"));
+ declearer.declareStream("error", new Fields("message"));
+ }
+
+ abstract void doPrepare(Map conf, TopologyContext topologyContext,
+ OutputCollector collector) throws IOException;
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/TelemetryAlertsBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/TelemetryAlertsBolt.java b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/TelemetryAlertsBolt.java
new file mode 100644
index 0000000..59c8784
--- /dev/null
+++ b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/TelemetryAlertsBolt.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apache.metron.alerts;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.configuration.Configuration;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Values;
+
+import com.google.common.cache.CacheBuilder;
+import com.apache.metron.alerts.interfaces.AlertsAdapter;
+import com.apache.metron.helpers.topology.ErrorGenerator;
+import com.apache.metron.json.serialization.JSONEncoderHelper;
+import com.apache.metron.metrics.MetricReporter;
+
+@SuppressWarnings("rawtypes")
+public class TelemetryAlertsBolt extends AbstractAlertBolt {
+
+ /**
+ * Use an adapter to tag existing telemetry messages with alerts. The list
+ * of available tagger adapters is located under
+ * com.apache.metron.tagging.adapters. At the time of the release the following
+ * adapters are available:
+ *
+ * <p>
+ * <ul>
+ * <li>RegexTagger = read a list or regular expressions and tag a message if
+ * they exist in a message
+ * <li>StaticAllTagger = tag each message with a static alert
+ * <ul>
+ * <p>
+ */
+ private static final long serialVersionUID = -2647123143398352020L;
+ private Properties metricProperties;
+ private JSONObject metricConfiguration;
+
+ // private AlertsCache suppressed_alerts;
+
+ /**
+ *
+ * @param tagger
+ * - tagger adapter for generating alert tags
+ * @return instance of bolt
+ */
+ public TelemetryAlertsBolt withAlertsAdapter(AlertsAdapter tagger) {
+ _adapter = tagger;
+ return this;
+ }
+
+ /**
+ *
+ * @param OutputFieldName
+ * - output name of the tuple coming out of this bolt
+ * @return - instance of this bolt
+ */
+ public TelemetryAlertsBolt withOutputFieldName(String OutputFieldName) {
+ this.OutputFieldName = OutputFieldName;
+ return this;
+ }
+
+ /**
+ *
+ * @param metricProperties
+ * - metric output to graphite
+ * @return - instance of this bolt
+ */
+ public TelemetryAlertsBolt withMetricProperties(Properties metricProperties) {
+ this.metricProperties = metricProperties;
+ return this;
+ }
+
+ /**
+ *
+ * @param identifier
+ * - the identifier tag for tagging telemetry messages with
+ * alerts out of this bolt
+ * @return - instance of this bolt
+ */
+
+ public TelemetryAlertsBolt withIdentifier(JSONObject identifier) {
+ this._identifier = identifier;
+ return this;
+ }
+
+ /**
+ * @param config
+ * A class for generating custom metrics into graphite
+ * @return Instance of this class
+ */
+
+ public TelemetryAlertsBolt withMetricConfiguration(Configuration config) {
+ this.metricConfiguration = JSONEncoderHelper.getJSON(config
+ .subset("com.apache.metron.metrics"));
+ return this;
+ }
+
+ /**
+ * @param MAX_CACHE_SIZE_OBJECTS_NUM
+ * Maximum size of cache before flushing
+ * @return Instance of this class
+ */
+
+ public TelemetryAlertsBolt withMaxCacheSize(int MAX_CACHE_SIZE_OBJECTS_NUM) {
+ _MAX_CACHE_SIZE_OBJECTS_NUM = MAX_CACHE_SIZE_OBJECTS_NUM;
+ return this;
+ }
+
+ /**
+ * @param MAX_TIME_RETAIN_MINUTES
+ * Maximum time to retain cached entry before expiring
+ * @return Instance of this class
+ */
+
+ public TelemetryAlertsBolt withMaxTimeRetain(int MAX_TIME_RETAIN_MINUTES) {
+ _MAX_TIME_RETAIN_MINUTES = MAX_TIME_RETAIN_MINUTES;
+ return this;
+ }
+
+ @Override
+ void doPrepare(Map conf, TopologyContext topologyContext,
+ OutputCollector collector) throws IOException {
+
+ cache = CacheBuilder.newBuilder().maximumSize(_MAX_CACHE_SIZE_OBJECTS_NUM)
+ .expireAfterWrite(_MAX_TIME_RETAIN_MINUTES, TimeUnit.MINUTES).build();
+
+ LOG.info("[Metron] Preparing TelemetryAlert Bolt...");
+
+ try {
+ _reporter = new MetricReporter();
+ _reporter.initialize(metricProperties, TelemetryAlertsBolt.class);
+ LOG.info("[Metron] Initialized metrics");
+ } catch (Exception e) {
+ LOG.info("[Metron] Could not initialize metrics");
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public void execute(Tuple tuple) {
+
+ LOG.trace("[Metron] Starting to process message for alerts");
+ JSONObject original_message = null;
+ String key = null;
+
+ try {
+
+ key = tuple.getStringByField("key");
+ original_message = (JSONObject) tuple.getValueByField("message");
+
+ if (original_message == null || original_message.isEmpty())
+ throw new Exception("Could not parse message from byte stream");
+
+ if(key == null)
+ throw new Exception("Key is not valid");
+
+ LOG.trace("[Metron] Received tuple: " + original_message);
+
+ JSONObject alerts_tag = new JSONObject();
+ Map<String, JSONObject> alerts_list = _adapter
+ .alert(original_message);
+ JSONArray uuid_list = new JSONArray();
+
+ if (alerts_list == null || alerts_list.isEmpty()) {
+ System.out.println("[Metron] No alerts detected in: "
+ + original_message);
+ _collector.ack(tuple);
+ _collector.emit("message", new Values(key, original_message));
+ } else {
+ for (String alert : alerts_list.keySet()) {
+ uuid_list.add(alert);
+
+ LOG.trace("[Metron] Checking alerts cache: " + alert);
+
+ if (cache.getIfPresent(alert) == null) {
+ System.out.println("[Metron]: Alert not found in cache: " + alert);
+
+ JSONObject global_alert = new JSONObject();
+ global_alert.putAll(_identifier);
+ global_alert.putAll(alerts_list.get(alert));
+ global_alert.put("timestamp", System.currentTimeMillis());
+ _collector.emit("alert", new Values(global_alert));
+
+ cache.put(alert, "");
+
+ } else
+ LOG.trace("Alert located in cache: " + alert);
+
+ LOG.debug("[Metron] Alerts are: " + alerts_list);
+
+ if (original_message.containsKey("alerts")) {
+ JSONArray already_triggered = (JSONArray) original_message
+ .get("alerts");
+
+ uuid_list.addAll(already_triggered);
+ LOG.trace("[Metron] Messages already had alerts...tagging more");
+ }
+
+ original_message.put("alerts", uuid_list);
+
+ LOG.debug("[Metron] Detected alerts: " + alerts_tag);
+
+ _collector.ack(tuple);
+ _collector.emit("message", new Values(key, original_message));
+
+ }
+
+ /*
+ * if (metricConfiguration != null) { emitCounter.inc();
+ * ackCounter.inc(); }
+ */
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOG.error("Failed to tag message :" + original_message);
+ e.printStackTrace();
+ _collector.fail(tuple);
+
+ /*
+ * if (metricConfiguration != null) { failCounter.inc(); }
+ */
+
+
+ JSONObject error = ErrorGenerator.generateErrorMessage(
+ "Alerts problem: " + original_message, e);
+ _collector.emit("error", new Values(error));
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/AbstractAlertAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/AbstractAlertAdapter.java b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/AbstractAlertAdapter.java
new file mode 100644
index 0000000..a3ab42a
--- /dev/null
+++ b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/AbstractAlertAdapter.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apache.metron.alerts.adapters;
+
+import java.io.Serializable;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.apache.metron.alerts.interfaces.AlertsAdapter;
+
+@SuppressWarnings("serial")
+public abstract class AbstractAlertAdapter implements AlertsAdapter, Serializable{
+
+ protected static final Logger _LOG = LoggerFactory
+ .getLogger(AbstractAlertAdapter.class);
+
+
+ protected Cache<String, String> cache;
+
+ protected String generateAlertId(String source_ip, String dst_ip,
+ int alert_type) {
+
+ String key = makeKey(source_ip, dst_ip, alert_type);
+
+ if (cache.getIfPresent(key) != null)
+ return cache.getIfPresent(key);
+
+ String new_UUID = System.currentTimeMillis() + "-" + UUID.randomUUID();
+
+ cache.put(key, new_UUID);
+ key = makeKey(dst_ip, source_ip, alert_type);
+ cache.put(key, new_UUID);
+
+ return new_UUID;
+
+ }
+
+ private String makeKey(String ip1, String ip2, int alert_type) {
+ return (ip1 + "-" + ip2 + "-" + alert_type);
+ }
+
+ protected void generateCache(int _MAX_CACHE_SIZE_OBJECTS_NUM, int _MAX_TIME_RETAIN_MINUTES)
+ {
+ cache = CacheBuilder.newBuilder().maximumSize(_MAX_CACHE_SIZE_OBJECTS_NUM)
+ .expireAfterWrite(_MAX_TIME_RETAIN_MINUTES, TimeUnit.MINUTES).build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/AllAlertAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/AllAlertAdapter.java b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/AllAlertAdapter.java
new file mode 100644
index 0000000..2a861ce
--- /dev/null
+++ b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/AllAlertAdapter.java
@@ -0,0 +1,274 @@
+package com.apache.metron.alerts.adapters;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.validator.routines.InetAddressValidator;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.json.simple.JSONObject;
+import org.apache.log4j.Logger;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.apache.metron.alerts.interfaces.AlertsAdapter;
+
+@SuppressWarnings("serial")
+public class AllAlertAdapter implements AlertsAdapter, Serializable {
+
+ HTableInterface blacklist_table;
+ HTableInterface whitelist_table;
+ InetAddressValidator ipvalidator = new InetAddressValidator();
+ String _whitelist_table_name;
+ String _blacklist_table_name;
+ String _quorum;
+ String _port;
+ String _topologyname;
+ Configuration conf = null;
+
+ Cache<String, String> cache;
+ String _topology_name;
+
+ Set<String> loaded_whitelist = new HashSet<String>();
+ Set<String> loaded_blacklist = new HashSet<String>();
+
+ protected static final Logger LOG = Logger
+ .getLogger(AllAlertAdapter.class);
+
+ public AllAlertAdapter(Map<String, String> config) {
+ try {
+ if(!config.containsKey("whitelist_table_name"))
+ throw new Exception("Whitelist table name is missing");
+
+ _whitelist_table_name = config.get("whitelist_table_name");
+
+ if(!config.containsKey("blacklist_table_name"))
+ throw new Exception("Blacklist table name is missing");
+
+ _blacklist_table_name = config.get("blacklist_table_name");
+
+ if(!config.containsKey("quorum"))
+ throw new Exception("Quorum name is missing");
+
+ _quorum = config.get("quorum");
+
+ if(!config.containsKey("port"))
+ throw new Exception("port name is missing");
+
+ _port = config.get("port");
+
+ if(!config.containsKey("_MAX_CACHE_SIZE_OBJECTS_NUM"))
+ throw new Exception("_MAX_CACHE_SIZE_OBJECTS_NUM name is missing");
+
+ int _MAX_CACHE_SIZE_OBJECTS_NUM = Integer.parseInt(config
+ .get("_MAX_CACHE_SIZE_OBJECTS_NUM"));
+
+ if(!config.containsKey("_MAX_TIME_RETAIN_MINUTES"))
+ throw new Exception("_MAX_TIME_RETAIN_MINUTES name is missing");
+
+ int _MAX_TIME_RETAIN_MINUTES = Integer.parseInt(config
+ .get("_MAX_TIME_RETAIN_MINUTES"));
+
+ cache = CacheBuilder.newBuilder().maximumSize(_MAX_CACHE_SIZE_OBJECTS_NUM)
+ .expireAfterWrite(_MAX_TIME_RETAIN_MINUTES, TimeUnit.MINUTES)
+ .build();
+ } catch (Exception e) {
+ System.out.println("Could not initialize Alerts Adapter");
+ e.printStackTrace();
+ System.exit(0);
+ }
+ }
+
+ @SuppressWarnings("resource")
+ @Override
+ public boolean initialize() {
+
+ conf = HBaseConfiguration.create();
+ //conf.set("hbase.zookeeper.quorum", _quorum);
+ //conf.set("hbase.zookeeper.property.clientPort", _port);
+
+ LOG.trace("[Metron] Connecting to hbase with conf:" + conf);
+ LOG.trace("[Metron] Whitelist table name: " + _whitelist_table_name);
+ LOG.trace("[Metron] Whitelist table name: " + _blacklist_table_name);
+ LOG.trace("[Metron] ZK Client/port: "
+ + conf.get("hbase.zookeeper.quorum") + " -> "
+ + conf.get("hbase.zookeeper.property.clientPort"));
+
+ try {
+
+ LOG.trace("[Metron] Attempting to connect to hbase");
+
+ HConnection connection = HConnectionManager.createConnection(conf);
+
+ LOG.trace("[Metron] CONNECTED TO HBASE");
+
+ HBaseAdmin hba = new HBaseAdmin(conf);
+
+ if (!hba.tableExists(_whitelist_table_name))
+ throw new Exception("Whitelist table doesn't exist");
+
+ if (!hba.tableExists(_blacklist_table_name))
+ throw new Exception("Blacklist table doesn't exist");
+
+ whitelist_table = new HTable(conf, _whitelist_table_name);
+
+ LOG.trace("[Metron] CONNECTED TO TABLE: " + _whitelist_table_name);
+ blacklist_table = new HTable(conf, _blacklist_table_name);
+ LOG.trace("[Metron] CONNECTED TO TABLE: " + _blacklist_table_name);
+
+ if (connection == null || whitelist_table == null
+ || blacklist_table == null)
+ throw new Exception("Unable to initialize hbase connection");
+
+ Scan scan = new Scan();
+
+ ResultScanner rs = whitelist_table.getScanner(scan);
+ try {
+ for (Result r = rs.next(); r != null; r = rs.next()) {
+ loaded_whitelist.add(Bytes.toString(r.getRow()));
+ }
+ } catch (Exception e) {
+ LOG.trace("[Metron] COULD NOT READ FROM HBASE");
+ e.printStackTrace();
+ } finally {
+ rs.close(); // always close the ResultScanner!
+ hba.close();
+ }
+ whitelist_table.close();
+
+ LOG.trace("[Metron] READ IN WHITELIST: " + loaded_whitelist.size());
+
+ System.out.println("LOADED WHITELIST IS: ");
+
+ for(String str: loaded_whitelist)
+ System.out.println("WHITELIST: " + str);
+
+ scan = new Scan();
+
+ rs = blacklist_table.getScanner(scan);
+ try {
+ for (Result r = rs.next(); r != null; r = rs.next()) {
+ loaded_blacklist.add(Bytes.toString(r.getRow()));
+ }
+ } catch (Exception e) {
+ LOG.trace("[Metron] COULD NOT READ FROM HBASE");
+ e.printStackTrace();
+ } finally {
+ rs.close(); // always close the ResultScanner!
+ hba.close();
+ }
+ blacklist_table.close();
+
+ LOG.trace("[Metron] READ IN WHITELIST: " + loaded_whitelist.size());
+
+ rs.close(); // always close the ResultScanner!
+ hba.close();
+
+ return true;
+ } catch (Exception e) {
+
+ e.printStackTrace();
+ }
+
+ return false;
+
+ }
+
+ @Override
+ public boolean refresh() throws Exception {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Map<String, JSONObject> alert(JSONObject raw_message) {
+
+ Map<String, JSONObject> alerts = new HashMap<String, JSONObject>();
+ JSONObject content = (JSONObject) raw_message.get("message");
+
+ JSONObject enrichment = null;
+
+ if (raw_message.containsKey("enrichment"))
+ enrichment = (JSONObject) raw_message.get("enrichment");
+
+ JSONObject alert = new JSONObject();
+
+
+
+ String source = "unknown";
+ String dest = "unknown";
+ String host = "unknown";
+
+ if (content.containsKey("ip_src_addr"))
+ {
+ source = content.get("ip_src_addr").toString();
+
+ if(RangeChecker.checkRange(loaded_whitelist, source))
+ host = source;
+ }
+
+ if (content.containsKey("ip_dst_addr"))
+ {
+ dest = content.get("ip_dst_addr").toString();
+
+ if(RangeChecker.checkRange(loaded_whitelist, dest))
+ host = dest;
+ }
+
+ alert.put("designated_host", host);
+ alert.put("description", content.get("original_string").toString());
+ alert.put("priority", "MED");
+
+ String alert_id = generateAlertId(source, dest, 0);
+
+ alert.put("alert_id", alert_id);
+ alerts.put(alert_id, alert);
+
+ alert.put("enrichment", enrichment);
+
+ return alerts;
+
+ }
+
+ @Override
+ public boolean containsAlertId(String alert) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ protected String generateAlertId(String source_ip, String dst_ip,
+ int alert_type) {
+
+ String key = makeKey(source_ip, dst_ip, alert_type);
+
+ if (cache.getIfPresent(key) != null)
+ return cache.getIfPresent(key);
+
+ String new_UUID = System.currentTimeMillis() + "-" + UUID.randomUUID();
+
+ cache.put(key, new_UUID);
+ key = makeKey(dst_ip, source_ip, alert_type);
+ cache.put(key, new_UUID);
+
+ return new_UUID;
+
+ }
+
+ private String makeKey(String ip1, String ip2, int alert_type) {
+ return (ip1 + "-" + ip2 + "-" + alert_type);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/CIFAlertsAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/CIFAlertsAdapter.java b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/CIFAlertsAdapter.java
new file mode 100644
index 0000000..0ea313f
--- /dev/null
+++ b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/CIFAlertsAdapter.java
@@ -0,0 +1,311 @@
+package com.apache.metron.alerts.adapters;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.validator.routines.InetAddressValidator;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.apache.metron.alerts.interfaces.AlertsAdapter;
+
+@SuppressWarnings("serial")
+public class CIFAlertsAdapter implements AlertsAdapter, Serializable {
+
+ String enrichment_tag;
+
+ HTableInterface blacklist_table;
+ HTableInterface whitelist_table;
+ InetAddressValidator ipvalidator = new InetAddressValidator();
+ String _whitelist_table_name;
+ String _blacklist_table_name;
+ String _quorum;
+ String _port;
+ String _topologyname;
+ Configuration conf = null;
+
+ Cache<String, String> cache;
+ String _topology_name;
+
+ Set<String> loaded_whitelist = new HashSet<String>();
+ Set<String> loaded_blacklist = new HashSet<String>();
+
+ protected static final Logger LOG = LoggerFactory
+ .getLogger(CIFAlertsAdapter.class);
+
+ public CIFAlertsAdapter(Map<String, String> config) {
+ try {
+
+ if (!config.containsKey("whitelist_table_name"))
+ throw new Exception("Whitelist table name is missing");
+
+ _whitelist_table_name = config.get("whitelist_table_name");
+
+ if (!config.containsKey("blacklist_table_name"))
+ throw new Exception("Blacklist table name is missing");
+
+ _blacklist_table_name = config.get("blacklist_table_name");
+
+ if (!config.containsKey("quorum"))
+ throw new Exception("Quorum name is missing");
+
+ _quorum = config.get("quorum");
+
+ if (!config.containsKey("port"))
+ throw new Exception("port name is missing");
+
+ _port = config.get("port");
+
+ if (!config.containsKey("_MAX_CACHE_SIZE_OBJECTS_NUM"))
+ throw new Exception("_MAX_CACHE_SIZE_OBJECTS_NUM name is missing");
+
+ int _MAX_CACHE_SIZE_OBJECTS_NUM = Integer.parseInt(config
+ .get("_MAX_CACHE_SIZE_OBJECTS_NUM"));
+
+ if (!config.containsKey("_MAX_TIME_RETAIN_MINUTES"))
+ throw new Exception("_MAX_TIME_RETAIN_MINUTES name is missing");
+
+ int _MAX_TIME_RETAIN_MINUTES = Integer.parseInt(config
+ .get("_MAX_TIME_RETAIN_MINUTES"));
+
+ cache = CacheBuilder.newBuilder().maximumSize(_MAX_CACHE_SIZE_OBJECTS_NUM)
+ .expireAfterWrite(_MAX_TIME_RETAIN_MINUTES, TimeUnit.MINUTES)
+ .build();
+
+ enrichment_tag = config.get("enrichment_tag");
+
+ } catch (Exception e) {
+ System.out.println("Could not initialize alerts adapter");
+ e.printStackTrace();
+ System.exit(0);
+ }
+ }
+
+ @SuppressWarnings("resource")
+ @Override
+ public boolean initialize() {
+
+ conf = HBaseConfiguration.create();
+ // conf.set("hbase.zookeeper.quorum", _quorum);
+ // conf.set("hbase.zookeeper.property.clientPort", _port);
+
+ LOG.trace("[Metron] Connecting to hbase with conf:" + conf);
+ LOG.trace("[Metron] Whitelist table name: " + _whitelist_table_name);
+ LOG.trace("[Metron] Whitelist table name: " + _blacklist_table_name);
+ LOG.trace("[Metron] ZK Client/port: "
+ + conf.get("hbase.zookeeper.quorum") + " -> "
+ + conf.get("hbase.zookeeper.property.clientPort"));
+
+ try {
+
+ LOG.trace("[Metron] Attempting to connect to hbase");
+
+ HConnection connection = HConnectionManager.createConnection(conf);
+
+ LOG.trace("[Metron] CONNECTED TO HBASE");
+
+ HBaseAdmin hba = new HBaseAdmin(conf);
+
+ if (!hba.tableExists(_whitelist_table_name))
+ throw new Exception("Whitelist table doesn't exist");
+
+ if (!hba.tableExists(_blacklist_table_name))
+ throw new Exception("Blacklist table doesn't exist");
+
+ whitelist_table = new HTable(conf, _whitelist_table_name);
+
+ LOG.trace("[Metron] CONNECTED TO TABLE: " + _whitelist_table_name);
+ blacklist_table = new HTable(conf, _blacklist_table_name);
+ LOG.trace("[Metron] CONNECTED TO TABLE: " + _blacklist_table_name);
+
+ if (connection == null || whitelist_table == null
+ || blacklist_table == null)
+ throw new Exception("Unable to initialize hbase connection");
+
+ Scan scan = new Scan();
+
+ ResultScanner rs = whitelist_table.getScanner(scan);
+ try {
+ for (Result r = rs.next(); r != null; r = rs.next()) {
+ loaded_whitelist.add(Bytes.toString(r.getRow()));
+ }
+ } catch (Exception e) {
+ LOG.trace("[Metron] COULD NOT READ FROM HBASE");
+ e.printStackTrace();
+ } finally {
+ rs.close(); // always close the ResultScanner!
+ hba.close();
+ }
+ whitelist_table.close();
+
+ LOG.trace("[Metron] READ IN WHITELIST: " + loaded_whitelist.size());
+
+ scan = new Scan();
+
+ rs = blacklist_table.getScanner(scan);
+ try {
+ for (Result r = rs.next(); r != null; r = rs.next()) {
+ loaded_blacklist.add(Bytes.toString(r.getRow()));
+ }
+ } catch (Exception e) {
+ LOG.trace("[Metron] COULD NOT READ FROM HBASE");
+ e.printStackTrace();
+ } finally {
+ rs.close(); // always close the ResultScanner!
+ hba.close();
+ }
+ blacklist_table.close();
+
+ LOG.trace("[Metron] READ IN WHITELIST: " + loaded_whitelist.size());
+
+ rs.close(); // always close the ResultScanner!
+ hba.close();
+
+ return true;
+ } catch (Exception e) {
+
+ e.printStackTrace();
+ }
+
+ return false;
+
+ }
+
+ @Override
+ public boolean refresh() throws Exception {
+ return true;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Map<String, JSONObject> alert(JSONObject raw_message) {
+
+ System.out.println("LOOKING FOR ENRICHMENT TAG: " + enrichment_tag);
+
+ Map<String, JSONObject> alerts = new HashMap<String, JSONObject>();
+ JSONObject content = (JSONObject) raw_message.get("message");
+
+ JSONObject enrichment = null;
+
+ if (raw_message.containsKey("enrichment"))
+ enrichment = (JSONObject) raw_message.get("enrichment");
+ else
+ return null;
+
+ if (enrichment.containsKey(enrichment_tag)) {
+
+ System.out.println("FOUND TAG: " + enrichment_tag);
+
+ JSONObject cif = (JSONObject) enrichment.get(enrichment_tag);
+
+ int cnt = 0;
+ Object enriched_key = null;
+
+ for (Object key : cif.keySet()) {
+ JSONObject tmp = (JSONObject) cif.get(key);
+ cnt = cnt + tmp.size();
+ if (tmp.size() > 0)
+ enriched_key = key;
+ }
+
+ if (cnt == 0) {
+ System.out.println("TAG HAS NO ELEMENTS");
+ return null;
+ }
+
+ JSONObject alert = new JSONObject();
+
+ String source = "unknown";
+ String dest = "unknown";
+ String host = "unknown";
+
+ if (content.containsKey("ip_src_addr")) {
+ source = content.get("ip_src_addr").toString();
+
+ if (RangeChecker.checkRange(loaded_whitelist, source))
+ host = source;
+ }
+
+ if (content.containsKey("ip_dst_addr")) {
+ dest = content.get("ip_dst_addr").toString();
+
+ if (RangeChecker.checkRange(loaded_whitelist, dest))
+ host = dest;
+ }
+
+ JSONObject cifQualifier = (JSONObject) cif.get(enriched_key);
+
+ alert.put("designated_host", host);
+ String description = new StringBuilder()
+ .append(host)
+ .append(" communicated with a host (")
+ .append(content.get(enriched_key).toString())
+ .append(") identified as ")
+ .append(cifQualifier.keySet().iterator().next().toString())
+ .append(" by CIF")
+ .toString();
+ alert.put("description", description);
+ alert.put("priority", "MED");
+
+ String alert_id = generateAlertId(source, dest, 0);
+
+ alert.put("alert_id", alert_id);
+ alerts.put(alert_id, alert);
+
+ alert.put("enrichment", enrichment);
+
+ return alerts;
+ } else {
+ System.out.println("DID NOT FIND TAG: " + enrichment_tag);
+ return null;
+ }
+
+ }
+
+ @Override
+ public boolean containsAlertId(String alert) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ protected String generateAlertId(String source_ip, String dst_ip,
+ int alert_type) {
+
+ String key = makeKey(source_ip, dst_ip, alert_type);
+
+ if (cache.getIfPresent(key) != null)
+ return cache.getIfPresent(key);
+
+ String new_UUID = System.currentTimeMillis() + "-" + UUID.randomUUID();
+
+ cache.put(key, new_UUID);
+ key = makeKey(dst_ip, source_ip, alert_type);
+ cache.put(key, new_UUID);
+
+ return new_UUID;
+
+ }
+
+ private String makeKey(String ip1, String ip2, int alert_type) {
+ return (ip1 + "-" + ip2 + "-" + alert_type);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/HbaseWhiteAndBlacklistAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/HbaseWhiteAndBlacklistAdapter.java b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/HbaseWhiteAndBlacklistAdapter.java
new file mode 100644
index 0000000..d5ca414
--- /dev/null
+++ b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/HbaseWhiteAndBlacklistAdapter.java
@@ -0,0 +1,466 @@
+package com.apache.metron.alerts.adapters;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.validator.routines.InetAddressValidator;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.apache.metron.alerts.interfaces.AlertsAdapter;
+
+public class HbaseWhiteAndBlacklistAdapter implements AlertsAdapter,
+ Serializable {
+
+ HTableInterface blacklist_table;
+ HTableInterface whitelist_table;
+ InetAddressValidator ipvalidator = new InetAddressValidator();
+ String _whitelist_table_name;
+ String _blacklist_table_name;
+ String _quorum;
+ String _port;
+ String _topologyname;
+ Configuration conf = null;
+
+ Cache<String, String> cache;
+ String _topology_name;
+
+ Set<String> loaded_whitelist = new HashSet<String>();
+ Set<String> loaded_blacklist = new HashSet<String>();
+
+ protected static final Logger LOG = LoggerFactory
+ .getLogger(HbaseWhiteAndBlacklistAdapter.class);
+
+ public HbaseWhiteAndBlacklistAdapter(Map<String, String> config) {
+
+ try {
+ if(!config.containsKey("whitelist_table_name"))
+ throw new Exception("Whitelist table name is missing");
+
+ _whitelist_table_name = config.get("whitelist_table_name");
+
+ if(!config.containsKey("blacklist_table_name"))
+ throw new Exception("Blacklist table name is missing");
+
+ _blacklist_table_name = config.get("blacklist_table_name");
+
+ if(!config.containsKey("quorum"))
+ throw new Exception("Quorum name is missing");
+
+ _quorum = config.get("quorum");
+
+ if(!config.containsKey("port"))
+ throw new Exception("port name is missing");
+
+ _port = config.get("port");
+
+ if(!config.containsKey("_MAX_CACHE_SIZE_OBJECTS_NUM"))
+ throw new Exception("_MAX_CACHE_SIZE_OBJECTS_NUM name is missing");
+
+ int _MAX_CACHE_SIZE_OBJECTS_NUM = Integer.parseInt(config
+ .get("_MAX_CACHE_SIZE_OBJECTS_NUM"));
+
+ if(!config.containsKey("_MAX_TIME_RETAIN_MINUTES"))
+ throw new Exception("_MAX_TIME_RETAIN_MINUTES name is missing");
+
+ int _MAX_TIME_RETAIN_MINUTES = Integer.parseInt(config
+ .get("_MAX_TIME_RETAIN_MINUTES"));
+
+ cache = CacheBuilder.newBuilder().maximumSize(_MAX_CACHE_SIZE_OBJECTS_NUM)
+ .expireAfterWrite(_MAX_TIME_RETAIN_MINUTES, TimeUnit.MINUTES)
+ .build();
+ } catch (Exception e) {
+ System.out.println("Could not initialize Alerts Adapter");
+ e.printStackTrace();
+ System.exit(0);
+ }
+
+ }
+
+ public boolean initialize() {
+
+ conf = HBaseConfiguration.create();
+ //conf.set("hbase.zookeeper.quorum", _quorum);
+ //conf.set("hbase.zookeeper.property.clientPort", _port);
+
+ LOG.trace("[Metron] Connecting to hbase with conf:" + conf);
+ LOG.trace("[Metron] Whitelist table name: " + _whitelist_table_name);
+ LOG.trace("[Metron] Whitelist table name: " + _blacklist_table_name);
+ LOG.trace("[Metron] ZK Client/port: "
+ + conf.get("hbase.zookeeper.quorum") + " -> "
+ + conf.get("hbase.zookeeper.property.clientPort"));
+
+ try {
+
+ LOG.trace("[Metron] Attempting to connect to hbase");
+
+ HConnection connection = HConnectionManager.createConnection(conf);
+
+ LOG.trace("[Metron] CONNECTED TO HBASE");
+
+ HBaseAdmin hba = new HBaseAdmin(conf);
+
+ if (!hba.tableExists(_whitelist_table_name))
+ throw new Exception("Whitelist table doesn't exist");
+
+ if (!hba.tableExists(_blacklist_table_name))
+ throw new Exception("Blacklist table doesn't exist");
+
+ whitelist_table = new HTable(conf, _whitelist_table_name);
+
+ LOG.trace("[Metron] CONNECTED TO TABLE: " + _whitelist_table_name);
+ blacklist_table = new HTable(conf, _blacklist_table_name);
+ LOG.trace("[Metron] CONNECTED TO TABLE: " + _blacklist_table_name);
+
+ if (connection == null || whitelist_table == null
+ || blacklist_table == null)
+ throw new Exception("Unable to initialize hbase connection");
+
+ Scan scan = new Scan();
+
+ ResultScanner rs = whitelist_table.getScanner(scan);
+ try {
+ for (Result r = rs.next(); r != null; r = rs.next()) {
+ loaded_whitelist.add(Bytes.toString(r.getRow()));
+ }
+ } catch (Exception e) {
+ LOG.trace("[Metron] COULD NOT READ FROM HBASE");
+ e.printStackTrace();
+ } finally {
+ rs.close(); // always close the ResultScanner!
+ hba.close();
+ }
+ whitelist_table.close();
+
+ LOG.trace("[Metron] READ IN WHITELIST: " + loaded_whitelist.size());
+
+ scan = new Scan();
+
+ rs = blacklist_table.getScanner(scan);
+ try {
+ for (Result r = rs.next(); r != null; r = rs.next()) {
+ loaded_blacklist.add(Bytes.toString(r.getRow()));
+ }
+ } catch (Exception e) {
+ LOG.trace("[Metron] COULD NOT READ FROM HBASE");
+ e.printStackTrace();
+ } finally {
+ rs.close(); // always close the ResultScanner!
+ hba.close();
+ }
+ blacklist_table.close();
+
+ LOG.trace("[Metron] READ IN WHITELIST: " + loaded_whitelist.size());
+
+ rs.close(); // always close the ResultScanner!
+ hba.close();
+
+ return true;
+ } catch (Exception e) {
+
+ e.printStackTrace();
+ }
+
+ return false;
+
+ }
+
+ protected String generateAlertId(String source_ip, String dst_ip,
+ int alert_type) {
+
+ String key = makeKey(source_ip, dst_ip, alert_type);
+
+ if (cache.getIfPresent(key) != null)
+ return cache.getIfPresent(key);
+
+ String new_UUID = System.currentTimeMillis() + "-" + UUID.randomUUID();
+
+ cache.put(key, new_UUID);
+ key = makeKey(dst_ip, source_ip, alert_type);
+ cache.put(key, new_UUID);
+
+ return new_UUID;
+
+ }
+
+ public boolean refresh() throws Exception {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ private String makeKey(String ip1, String ip2, int alert_type) {
+ return (ip1 + "-" + ip2 + "-" + alert_type);
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map<String, JSONObject> alert(JSONObject raw_message) {
+
+ Map<String, JSONObject> alerts = new HashMap<String, JSONObject>();
+
+ JSONObject content = (JSONObject) raw_message.get("message");
+ JSONObject enrichment = null;
+
+ if (raw_message.containsKey("enrichment"))
+ enrichment = (JSONObject) raw_message.get("enrichment");
+
+ if (!content.containsKey("ip_src_addr")
+ || !content.containsKey("ip_dst_addr")) {
+
+ int alert_type = 0;
+
+ JSONObject alert = new JSONObject();
+
+ alert.put("title", "IP Check Error Type: : " + alert_type);
+ alert.put("priority", "1");
+ alert.put("type", "error");
+ alert.put("designated_host", "Uknown");
+ alert.put("source", "NA");
+ alert.put("dest", "NA");
+ alert.put("body", "Source or destination IP is missing");
+
+ String alert_id = UUID.randomUUID().toString();
+
+ alert.put("reference_id", alert_id);
+ alerts.put(alert_id, alert);
+
+ if (enrichment != null)
+ alert.put("enrichment", enrichment);
+
+ LOG.trace("[Metron] Returning alert: " + alerts);
+
+ return alerts;
+
+ }
+
+ String source_ip = content.get("ip_src_addr").toString();
+ String dst_ip = content.get("ip_dst_addr").toString();
+
+ if (source_ip == null && dst_ip == null) {
+
+ int alert_type = 1;
+
+ JSONObject alert = new JSONObject();
+
+ alert.put("title", "IP Check Error Type: : " + alert_type);
+ alert.put("priority", "1");
+ alert.put("type", "error");
+ alert.put("designated_host", "Uknown");
+ alert.put("source", source_ip);
+ alert.put("dest", dst_ip);
+ alert.put(
+ "body",
+ "This communication does not contain a source or destination IP string. Communication between two IPs: "
+ + source_ip + " -> " + dst_ip);
+
+ String alert_id = generateAlertId(source_ip, dst_ip, alert_type);
+
+ alert.put("reference_id", alert_id);
+ alerts.put(alert_id, alert);
+ if (enrichment != null)
+ alert.put("enrichment", enrichment);
+
+ LOG.trace("[Metron] Returning alert: " + alerts);
+
+ return alerts;
+
+ }
+
+ if (!ipvalidator.isValidInet4Address(source_ip)
+ && !ipvalidator.isValidInet4Address(dst_ip)) {
+ int alert_type = 2;
+
+ JSONObject alert = new JSONObject();
+
+ alert.put("title", "IP Check Error Type: : " + alert_type);
+ alert.put("priority", "1");
+ alert.put("type", "error");
+ alert.put("designated_host", "Uknown");
+ alert.put("source", source_ip);
+ alert.put("dest", dst_ip);
+ alert.put(
+ "content",
+ "This communication contains souce and destination IP strings, but these strings are not valid. Communication between two IPs: "
+ + source_ip + " -> " + dst_ip);
+
+ String alert_id = generateAlertId(source_ip, dst_ip, alert_type);
+
+ alert.put("reference_id", alert_id);
+ alerts.put(alert_id, alert);
+ if (enrichment != null)
+ alert.put("enrichment", enrichment);
+
+ LOG.trace("[Metron] Returning alert: " + alerts);
+
+ return alerts;
+
+ }
+
+ String designated_host = null;
+
+ if (loaded_whitelist.contains(source_ip))
+ designated_host = source_ip;
+ else if (loaded_whitelist.contains(dst_ip))
+ designated_host = dst_ip;
+
+ if (designated_host == null) {
+ int alert_type = 3;
+
+ JSONObject alert = new JSONObject();
+
+ alert.put("title", "IP Check Error Type: : " + alert_type);
+ alert.put("priority", "1");
+ alert.put("type", "error");
+ alert.put("designated_host", "Uknown");
+ alert.put("source", source_ip);
+ alert.put("dest", dst_ip);
+ alert.put(
+ "content",
+ "This communication does not contain a source or a destination IP that is in the white list. Communication between two IPs: "
+ + source_ip + " -> " + dst_ip);
+
+ String alert_id = generateAlertId(source_ip, dst_ip, alert_type);
+
+ alert.put("reference_id", alert_id);
+ alerts.put(alert_id, alert);
+ if (enrichment != null)
+ alert.put("enrichment", enrichment);
+
+ LOG.trace("[Metron] Returning alert: " + alerts);
+
+ return alerts;
+
+ }
+
+ if (source_ip.equals(designated_host)
+ && !ipvalidator.isValidInet4Address(dst_ip)) {
+ int alert_type = 4;
+
+ JSONObject alert = new JSONObject();
+
+ alert.put("title", "IP Check Error Type: : " + alert_type);
+ alert.put("priority", "1");
+ alert.put("type", "error");
+ alert.put("designated_host", designated_host);
+ alert.put("source", source_ip);
+ alert.put("dest", dst_ip);
+ alert.put(
+ "content",
+ "This communication contains an IP that is not valid. Communication between two IPs: "
+ + source_ip + " -> " + dst_ip);
+
+ String alert_id = generateAlertId(source_ip, dst_ip, alert_type);
+
+ alert.put("reference_id", alert_id);
+ alerts.put(alert_id, alert);
+ if (enrichment != null)
+ alert.put("enrichment", enrichment);
+
+ }
+
+ if (dst_ip.equals(designated_host)
+ && !ipvalidator.isValidInet4Address(source_ip)) {
+ int alert_type = 5;
+
+ JSONObject alert = new JSONObject();
+
+ alert.put("title", "IP Check Error Type: : " + alert_type);
+ alert.put("priority", "1");
+ alert.put("type", "error");
+ alert.put("designated_host", designated_host);
+ alert.put("source", source_ip);
+ alert.put("dest", dst_ip);
+ alert.put(
+ "content",
+ "This communication contains IP that is not valid. Communication between two IPs: "
+ + source_ip + " -> " + dst_ip);
+
+ String alert_id = generateAlertId(source_ip, dst_ip, alert_type);
+
+ alert.put("reference_id", alert_id);
+ alerts.put(alert_id, alert);
+ if (enrichment != null)
+ alert.put("enrichment", enrichment);
+
+ }
+
+ if (loaded_blacklist.contains(source_ip)) {
+ int alert_type = 6;
+
+ JSONObject alert = new JSONObject();
+
+ alert.put("title", "IP Check Error Type: : " + alert_type);
+ alert.put("priority", "1");
+ alert.put("type", "error");
+ alert.put("designated_host", designated_host);
+ alert.put("source", source_ip);
+ alert.put("dest", dst_ip);
+ alert.put(
+ "content",
+ "This communication contains IP that is black listed. Communication between two IPs: "
+ + source_ip + " -> " + dst_ip);
+
+ String alert_id = generateAlertId(source_ip, dst_ip, alert_type);
+
+ alert.put("reference_id", alert_id);
+ alerts.put(alert_id, alert);
+ if (enrichment != null)
+ alert.put("enrichment", enrichment);
+
+ }
+
+ if (loaded_blacklist.contains(dst_ip)) {
+ int alert_type = 7;
+
+ JSONObject alert = new JSONObject();
+
+ alert.put("title", "IP Check Error Type: : " + alert_type);
+ alert.put("priority", "1");
+ alert.put("type", "error");
+ alert.put("designated_host", designated_host);
+ alert.put("source", source_ip);
+ alert.put("dest", dst_ip);
+ alert.put(
+ "content",
+ "This communication contains IP that is black listed. Communication between two IPs: "
+ + source_ip + " -> " + dst_ip);
+
+ String alert_id = generateAlertId(source_ip, dst_ip, alert_type);
+
+ alert.put("reference_id", alert_id);
+ alerts.put(alert_id, alert);
+ if (enrichment != null)
+ alert.put("enrichment", enrichment);
+
+ }
+
+ if (alerts.isEmpty())
+ return null;
+ else
+ return alerts;
+ }
+
+ public boolean containsAlertId(String alert) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/KeywordsAlertAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/KeywordsAlertAdapter.java b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/KeywordsAlertAdapter.java
new file mode 100644
index 0000000..4db45f8
--- /dev/null
+++ b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/KeywordsAlertAdapter.java
@@ -0,0 +1,274 @@
+package com.apache.metron.alerts.adapters;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.validator.routines.InetAddressValidator;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.json.simple.JSONObject;
+import org.apache.log4j.Logger;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+
+import com.apache.metron.alerts.interfaces.AlertsAdapter;
+
+public class KeywordsAlertAdapter extends AbstractAlertAdapter {
+
+ HTableInterface blacklist_table;
+ HTableInterface whitelist_table;
+ InetAddressValidator ipvalidator = new InetAddressValidator();
+ String _whitelist_table_name;
+ String _blacklist_table_name;
+ String _quorum;
+ String _port;
+ String _topologyname;
+ Configuration conf = null;
+
+ String _topology_name;
+
+ Set<String> loaded_whitelist = new HashSet<String>();
+ Set<String> loaded_blacklist = new HashSet<String>();
+
+ List<String> keywordList;
+ List<String> keywordExceptionList;
+
+ protected static final Logger LOG = Logger.getLogger(AllAlertAdapter.class);
+
+ public KeywordsAlertAdapter(Map<String, String> config) {
+ try {
+
+ if(!config.containsKey("keywords"))
+ throw new Exception("Keywords are missing");
+
+ keywordList = Arrays.asList(config.get("keywords").split("\\|"));
+
+ if( config.containsKey("exceptions")) {
+ keywordExceptionList = Arrays.asList(config.get("exceptions").split("\\|"));
+ } else {
+ keywordExceptionList = new ArrayList<String>();
+ }
+
+ if(!config.containsKey("whitelist_table_name"))
+ throw new Exception("Whitelist table name is missing");
+
+ _whitelist_table_name = config.get("whitelist_table_name");
+
+ if(!config.containsKey("blacklist_table_name"))
+ throw new Exception("Blacklist table name is missing");
+
+ _blacklist_table_name = config.get("blacklist_table_name");
+
+ if(!config.containsKey("quorum"))
+ throw new Exception("Quorum name is missing");
+
+ _quorum = config.get("quorum");
+
+ if(!config.containsKey("port"))
+ throw new Exception("port name is missing");
+
+ _port = config.get("port");
+
+ if(!config.containsKey("_MAX_CACHE_SIZE_OBJECTS_NUM"))
+ throw new Exception("_MAX_CACHE_SIZE_OBJECTS_NUM name is missing");
+
+ int _MAX_CACHE_SIZE_OBJECTS_NUM = Integer.parseInt(config
+ .get("_MAX_CACHE_SIZE_OBJECTS_NUM"));
+
+ if(!config.containsKey("_MAX_TIME_RETAIN_MINUTES"))
+ throw new Exception("_MAX_TIME_RETAIN_MINUTES name is missing");
+
+ int _MAX_TIME_RETAIN_MINUTES = Integer.parseInt(config
+ .get("_MAX_TIME_RETAIN_MINUTES"));
+
+ generateCache(_MAX_CACHE_SIZE_OBJECTS_NUM, _MAX_TIME_RETAIN_MINUTES);
+
+ } catch (Exception e) {
+ System.out.println("Could not initialize Alerts Adapter");
+ e.printStackTrace();
+ System.exit(0);
+ }
+ }
+
+ @Override
+ public boolean initialize() {
+ conf = HBaseConfiguration.create();
+ //conf.set("hbase.zookeeper.quorum", _quorum);
+ //conf.set("hbase.zookeeper.property.clientPort", _port);
+
+ LOG.trace("[Metron] Connecting to hbase with conf:" + conf);
+ LOG.trace("[Metron] Whitelist table name: " + _whitelist_table_name);
+ LOG.trace("[Metron] Whitelist table name: " + _blacklist_table_name);
+ LOG.trace("[Metron] ZK Client/port: "
+ + conf.get("hbase.zookeeper.quorum") + " -> "
+ + conf.get("hbase.zookeeper.property.clientPort"));
+
+ try {
+
+ LOG.trace("[Metron] Attempting to connect to hbase");
+
+ HConnection connection = HConnectionManager.createConnection(conf);
+
+ LOG.trace("[Metron] CONNECTED TO HBASE");
+
+ HBaseAdmin hba = new HBaseAdmin(conf);
+
+ if (!hba.tableExists(_whitelist_table_name))
+ throw new Exception("Whitelist table doesn't exist");
+
+ if (!hba.tableExists(_blacklist_table_name))
+ throw new Exception("Blacklist table doesn't exist");
+
+ whitelist_table = new HTable(conf, _whitelist_table_name);
+
+ LOG.trace("[Metron] CONNECTED TO TABLE: " + _whitelist_table_name);
+ blacklist_table = new HTable(conf, _blacklist_table_name);
+ LOG.trace("[Metron] CONNECTED TO TABLE: " + _blacklist_table_name);
+
+ if (connection == null || whitelist_table == null
+ || blacklist_table == null)
+ throw new Exception("Unable to initialize hbase connection");
+
+ Scan scan = new Scan();
+
+ ResultScanner rs = whitelist_table.getScanner(scan);
+ try {
+ for (Result r = rs.next(); r != null; r = rs.next()) {
+ loaded_whitelist.add(Bytes.toString(r.getRow()));
+ }
+ } catch (Exception e) {
+ LOG.trace("[Metron] COULD NOT READ FROM HBASE");
+ e.printStackTrace();
+ } finally {
+ rs.close(); // always close the ResultScanner!
+ hba.close();
+ }
+ whitelist_table.close();
+
+ LOG.trace("[Metron] READ IN WHITELIST: " + loaded_whitelist.size());
+
+ System.out.println("LOADED WHITELIST IS: ");
+
+ for(String str: loaded_whitelist)
+ System.out.println("WHITELIST: " + str);
+
+ scan = new Scan();
+
+ rs = blacklist_table.getScanner(scan);
+ try {
+ for (Result r = rs.next(); r != null; r = rs.next()) {
+ loaded_blacklist.add(Bytes.toString(r.getRow()));
+ }
+ } catch (Exception e) {
+ LOG.trace("[Metron] COULD NOT READ FROM HBASE");
+ e.printStackTrace();
+ } finally {
+ rs.close(); // always close the ResultScanner!
+ hba.close();
+ }
+ blacklist_table.close();
+
+ LOG.trace("[Metron] READ IN WHITELIST: " + loaded_whitelist.size());
+
+ rs.close(); // always close the ResultScanner!
+ hba.close();
+
+ return true;
+ } catch (Exception e) {
+
+ e.printStackTrace();
+ }
+
+ return false;
+ }
+
+ @Override
+ public boolean refresh() throws Exception {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean containsAlertId(String alert) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public Map<String, JSONObject> alert(JSONObject raw_message) {
+
+ Map<String, JSONObject> alerts = new HashMap<String, JSONObject>();
+ JSONObject content = (JSONObject) raw_message.get("message");
+
+ JSONObject enrichment = null;
+ if (raw_message.containsKey("enrichment"))
+ enrichment = (JSONObject) raw_message.get("enrichment");
+
+ for (String keyword : keywordList) {
+ if (content.toString().contains(keyword)) {
+
+ //check it doesn't have an "exception" keyword in it
+ for (String exception : keywordExceptionList) {
+ if (content.toString().contains(exception)) {
+ LOG.info("[Metron] KeywordAlertsAdapter: Omitting alert due to exclusion: " + exception);
+ return null;
+ }
+ }
+
+ LOG.info("[Metron] KeywordAlertsAdapter: Found match for " + keyword);
+ JSONObject alert = new JSONObject();
+
+ String source = "unknown";
+ String dest = "unknown";
+ String host = "unknown";
+
+ if (content.containsKey("ip_src_addr"))
+ {
+ source = content.get("ip_src_addr").toString();
+
+ if(RangeChecker.checkRange(loaded_whitelist, source))
+ host = source;
+ }
+
+ if (content.containsKey("ip_dst_addr"))
+ {
+ dest = content.get("ip_dst_addr").toString();
+
+ if(RangeChecker.checkRange(loaded_whitelist, dest))
+ host = dest;
+ }
+
+ alert.put("designated_host", host);
+ alert.put("description", content.get("original_string").toString());
+ alert.put("priority", "MED");
+
+ String alert_id = generateAlertId(source, dest, 0);
+
+ alert.put("alert_id", alert_id);
+ alerts.put(alert_id, alert);
+
+ alert.put("enrichment", enrichment);
+
+ return alerts;
+ }
+ }
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/RangeChecker.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/RangeChecker.java b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/RangeChecker.java
new file mode 100644
index 0000000..2b37401
--- /dev/null
+++ b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/adapters/RangeChecker.java
@@ -0,0 +1,23 @@
+package com.apache.metron.alerts.adapters;
+
+import java.util.Set;
+
+import org.apache.commons.net.util.SubnetUtils;
+
+public class RangeChecker {
+
+ static boolean checkRange(Set<String> CIDR_networks, String ip) {
+ for (String network : CIDR_networks) {
+
+ System.out.println("Looking at range: " + network + " and ip " + ip);
+ SubnetUtils utils = new SubnetUtils(network);
+ if(utils.getInfo().isInRange(ip)) {
+ System.out.println(ip + " in range " + network);
+ return true;
+ }
+ }
+
+ //no matches
+ return false;
+ }
+}