You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/03/01 20:29:56 UTC
[1/9] incubator-metron git commit: METRON-56 Create unified
enrichment topology (merrimanr via cestella) closes
apache/incubator-metron#33
Repository: incubator-metron
Updated Branches:
refs/heads/master a7e3879ed -> 3be012db9
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/SampleParsed/SnortParsed
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/SampleParsed/SnortParsed b/metron-streaming/Metron-Topologies/src/main/resources/SampleParsed/SnortParsed
new file mode 100644
index 0000000..4b74794
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/SampleParsed/SnortParsed
@@ -0,0 +1,3 @@
+{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","dst":"10.0.2.15","dstport":"22","ethsrc":"52:54:00:12:35:02","tcpseq":"0x9AFF3D7","dgmlen":"64","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0xC8761D52","original_string":"01\/27-16:01:04.877970 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,,","icmpcode":"","tos":"0","id":"59677","timestamp":1453932941970,"ethdst":"08:00:27:7F:93:2D","src":"10.0.2.2","ttl":"64","source.type":"test","ethlen":"0x4E","iplen":"65536","icmptype":"","proto":"TCP","srcport":"56642","tcpflags":"***AP***","sig_id":"12","sig_generator":"129"}
+{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","dst":"10.0.2.15","dstport":"50895","ethsrc":"52:54:00:12:35:02","tcpseq":"0xDB45F7A","dgmlen":"96","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0x7701DD5B","original_string":"02\/22-15:56:48.612494 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0x6E,***AP***,0xDB45F7A,0x7701DD5B,,0xFFFF,64,0,16785,96,98304,,,,","icmpcode":"","tos":"0","id":"16785","timestamp":1456178820494,"ethdst":"08:00:27:7F:93:2D","src":"96.44.142.5","ttl":"64","source.type":"test","ethlen":"0x6E","iplen":"98304","icmptype":"","proto":"TCP","srcport":"80","tcpflags":"***AP***","sig_id":"12","sig_generator":"129"}
+{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","dst":"10.0.2.15","dstport":"50895","ethsrc":"52:54:00:12:35:02","tcpseq":"0xDB508F2","dgmlen":"152","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0x7701DD5B","original_string":"02\/22-15:56:48.616775 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0xA6,***AP***,0xDB508F2,0x7701DD5B,,0xFFFF,64,0,16824,152,155648,,,,","icmpcode":"","tos":"0","id":"16824","timestamp":1456178824775,"ethdst":"08:00:27:7F:93:2D","src":"96.44.142.5","ttl":"64","source.type":"test","ethlen":"0xA6","iplen":"155648","icmptype":"","proto":"TCP","srcport":"80","tcpflags":"***AP***","sig_id":"12","sig_generator":"129"}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/SampleParsed/YafExampleParsed
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/SampleParsed/YafExampleParsed b/metron-streaming/Metron-Topologies/src/main/resources/SampleParsed/YafExampleParsed
new file mode 100644
index 0000000..57f07b1
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/SampleParsed/YafExampleParsed
@@ -0,0 +1,10 @@
+{"iflags":"AS","uflags":0,"isn":"22efa001","dip":"10.0.2.15","dp":39468,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"sip":"216.21.170.221","tag":0,"rtag":0,"sp":80,"timestamp":1453994988512,"app":0,"oct":44,"end_reason":"idle","risn":0,"end_time":1453994988512,"source.type":"yaf","start_time":1453994988512,"riflags":0,"rtt":"0.000","proto":6}
+{"iflags":"A","uflags":0,"isn":10000000,"dip":"10.0.2.3","dp":53,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"sip":"10.0.2.15","tag":0,"rtag":0,"sp":37299,"timestamp":1453994988502,"app":0,"oct":56,"end_reason":"idle","risn":0,"end_time":1453994988502,"source.type":"yaf","start_time":1453994988502,"riflags":0,"rtt":"0.000","proto":17}
+{"iflags":"A","uflags":0,"isn":0,"dip":"10.0.2.15","dp":37299,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"sip":"10.0.2.3","tag":0,"rtag":0,"sp":53,"timestamp":1453994988504,"app":0,"oct":312,"end_reason":"idle","risn":0,"end_time":1453994988504,"source.type":"yaf","start_time":1453994988504,"riflags":0,"rtt":"0.000","proto":17}
+{"iflags":"A","uflags":0,"isn":0,"dip":"10.0.2.3","dp":53,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"sip":"10.0.2.15","tag":0,"rtag":0,"sp":56303,"timestamp":1453994988504,"app":0,"oct":56,"end_reason":"idle","risn":0,"end_time":1453994988504,"source.type":"yaf","start_time":1453994988504,"riflags":0,"rtt":"0.000","proto":17}
+{"iflags":"A","uflags":0,"isn":0,"dip":"10.0.2.15","dp":56303,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"sip":"10.0.2.3","tag":0,"rtag":0,"sp":53,"timestamp":1453994988506,"app":0,"oct":84,"end_reason":"idle","risn":0,"end_time":1453994988506,"source.type":"yaf","start_time":1453994988506,"riflags":0,"rtt":"0.000","proto":17}
+{"iflags":"S","uflags":0,"isn":"58c52fca","dip":"216.21.170.221","dp":80,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"sip":"10.0.2.15","tag":0,"rtag":0,"sp":39468,"timestamp":1453994988508,"app":0,"oct":60,"end_reason":"idle","risn":0,"end_time":1453994988508,"source.type":"yaf","start_time":1453994988508,"riflags":0,"rtt":"0.000","proto":6}
+{"iflags":"A","uflags":0,"isn":"58c52fcb","dip":"216.21.170.221","dp":80,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle ","pkt":1,"ruflags":0,"roct":0,"sip":"10.0.2.15","tag":0,"rtag":0,"sp":39468,"timestamp":1453994988512,"app":0,"oct":40,"end_reason":"idle ","risn":0,"end_time":1453994988512,"source.type":"yaf","start_time":1453994988512,"riflags":0,"rtt":"0.000","proto":6}
+{"iflags":"AP","uflags":0,"isn":"58c52fcb","dip":"216.21.170.221","dp":80,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle ","pkt":1,"ruflags":0,"roct":0,"sip":"10.0.2.15","tag":0,"rtag":0,"sp":39468,"timestamp":1453994988512,"app":0,"oct":148,"end_reason":"idle ","risn":0,"end_time":1453994988512,"source.type":"yaf","start_time":1453994988512,"riflags":0,"rtt":"0.000","proto":6}
+{"iflags":"A","uflags":0,"isn":"22efa002","dip":"10.0.2.15","dp":39468,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle ","pkt":1,"ruflags":0,"roct":0,"sip":"216.21.170.221","tag":0,"rtag":0,"sp":80,"timestamp":1453994988512,"app":0,"oct":40,"end_reason":"idle ","risn":0,"end_time":1453994988512,"source.type":"yaf","start_time":1453994988512,"riflags":0,"rtt":"0.000","proto":6}
+{"iflags":"AP","uflags":0,"isn":"22efa002","dip":"10.0.2.15","dp":39468,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"sip":"216.21.170.221","tag":0,"rtag":0,"sp":80,"timestamp":1453994988562,"app":0,"oct":604,"end_reason":"idle","risn":0,"end_time":1453994988562,"source.type":"yaf","start_time":1453994988562,"riflags":0,"rtt":"0.000","proto":6}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/EnrichmentIntegrationTest.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
new file mode 100644
index 0000000..ef1318e
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration;
+
+import com.google.common.base.Function;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.metron.Constants;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
+import org.apache.metron.integration.util.TestUtils;
+import org.apache.metron.integration.util.UnitTestHelper;
+import org.apache.metron.integration.util.integration.ComponentRunner;
+import org.apache.metron.integration.util.integration.Processor;
+import org.apache.metron.integration.util.integration.ReadinessState;
+import org.apache.metron.integration.util.integration.components.ElasticSearchComponent;
+import org.apache.metron.integration.util.integration.components.FluxTopologyComponent;
+import org.apache.metron.integration.util.integration.components.KafkaWithZKComponent;
+import org.apache.metron.integration.util.mock.MockHTable;
+import org.apache.metron.integration.util.threatintel.ThreatIntelHelper;
+import org.apache.metron.reference.lookup.LookupKV;
+import org.apache.metron.utils.SourceConfigUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+public class EnrichmentIntegrationTest {
+
+ private String fluxPath = "src/main/resources/Metron_Configs/topologies/enrichment/test.yaml";
+ private String indexDir = "target/elasticsearch";
+ private String sampleParsedPath = "src/main/resources/SampleParsed/YafExampleParsed";
+ private String sampleIndexedPath = "src/main/resources/SampleIndexed/YafIndexed";
+ private Map<String, String> sourceConfigs = new HashMap<>();
+
+ public static class Provider implements TableProvider, Serializable {
+ MockHTable.Provider provider = new MockHTable.Provider();
+ @Override
+ public HTableInterface getTable(Configuration config, String tableName) throws IOException {
+ return provider.getTable(config, tableName);
+ }
+ }
+
+
+ @Test
+ public void test() throws Exception {
+ final String dateFormat = "yyyy.MM.dd.hh";
+ final String index = "yaf_" + new SimpleDateFormat(dateFormat).format(new Date());
+ String yafConfig = "{\n" +
+ " \"index\": \"yaf\",\n" +
+ " \"batchSize\": 5,\n" +
+ " \"enrichmentFieldMap\":\n" +
+ " {\n" +
+ " \"geo\": [\"sip\", \"dip\"],\n" +
+ " \"host\": [\"sip\", \"dip\"]\n" +
+ " },\n" +
+ " \"threatIntelFieldMap\":\n" +
+ " {\n" +
+ " \"ip\": [\"sip\", \"dip\"]\n" +
+ " }\n" +
+ "}";
+ sourceConfigs.put("yaf", yafConfig);
+ final List<byte[]> inputMessages = TestUtils.readSampleData(sampleParsedPath);
+ final String cf = "cf";
+ final String trackerHBaseTable = "tracker";
+ final String ipThreatIntelTable = "ip_threat_intel";
+ final Properties topologyProperties = new Properties() {{
+ setProperty("org.apache.metron.enrichment.host.known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"},\n" +
+ "{\"ip\":\"10.1.128.237\", \"local\":\"UNKNOWN\", \"type\":\"unknown\", \"asset_value\" : \"important\"},\n" +
+ "{\"ip\":\"10.60.10.254\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"},\n" +
+ "{\"ip\":\"10.0.2.15\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}]");
+ setProperty("hbase.provider.impl","" + Provider.class.getName());
+ setProperty("threat.intel.tracker.table", trackerHBaseTable);
+ setProperty("threat.intel.tracker.cf", cf);
+ setProperty("threat.intel.ip.table", ipThreatIntelTable);
+ setProperty("threat.intel.ip.cf", cf);
+ setProperty("es.clustername", "metron");
+ setProperty("es.port", "9300");
+ setProperty("es.ip", "localhost");
+ setProperty("index.date.format", dateFormat);
+ }};
+ final KafkaWithZKComponent kafkaComponent = new KafkaWithZKComponent().withTopics(new ArrayList<KafkaWithZKComponent.Topic>() {{
+ add(new KafkaWithZKComponent.Topic(Constants.ENRICHMENT_TOPIC, 1));
+ }})
+ .withPostStartCallback(new Function<KafkaWithZKComponent, Void>() {
+ @Nullable
+ @Override
+ public Void apply(@Nullable KafkaWithZKComponent kafkaWithZKComponent) {
+ topologyProperties.setProperty("kafka.zk", kafkaWithZKComponent.getZookeeperConnect());
+ try {
+ for(String sourceType: sourceConfigs.keySet()) {
+ SourceConfigUtils.writeToZookeeper(sourceType, sourceConfigs.get(sourceType).getBytes(), kafkaWithZKComponent.getZookeeperConnect());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+ });
+
+ ElasticSearchComponent esComponent = new ElasticSearchComponent.Builder()
+ .withHttpPort(9211)
+ .withIndexDir(new File(indexDir))
+ .build();
+
+ //create MockHBaseTables
+ final MockHTable trackerTable = (MockHTable)MockHTable.Provider.addToCache(trackerHBaseTable, cf);
+ final MockHTable ipTable = (MockHTable)MockHTable.Provider.addToCache(ipThreatIntelTable, cf);
+ ThreatIntelHelper.INSTANCE.load(ipTable, cf, new ArrayList<LookupKV<ThreatIntelKey, ThreatIntelValue>>(){{
+ add(new LookupKV<>(new ThreatIntelKey("10.0.2.3"), new ThreatIntelValue(new HashMap<String, String>())));
+ }});
+
+ FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder()
+ .withTopologyLocation(new File(fluxPath))
+ .withTopologyName("test")
+ .withTopologyProperties(topologyProperties)
+ .build();
+
+ UnitTestHelper.verboseLogging();
+ ComponentRunner runner = new ComponentRunner.Builder()
+ .withComponent("kafka", kafkaComponent)
+ .withComponent("elasticsearch", esComponent)
+ .withComponent("storm", fluxComponent)
+ .withTimeBetweenAttempts(10000)
+ .build();
+ runner.start();
+ fluxComponent.submitTopology();
+ kafkaComponent.writeMessages(Constants.ENRICHMENT_TOPIC, inputMessages);
+ List<Map<String, Object>> docs =
+ runner.process(new Processor<List<Map<String, Object>>> () {
+ List<Map<String, Object>> docs = null;
+ public ReadinessState process(ComponentRunner runner){
+ ElasticSearchComponent elasticSearchComponent = runner.getComponent("elasticsearch", ElasticSearchComponent.class);
+ if(elasticSearchComponent.hasIndex(index)) {
+ try {
+ docs = elasticSearchComponent.getAllIndexedDocs(index, "yaf");
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to retrieve indexed documents.", e);
+ }
+ if(docs.size() < inputMessages.size()) {
+ return ReadinessState.NOT_READY;
+ }
+ else {
+ return ReadinessState.READY;
+ }
+ }
+ else {
+ return ReadinessState.NOT_READY;
+ }
+ }
+
+ public List<Map<String, Object>> getResult() {
+ return docs;
+ }
+ });
+
+ List<byte[]> sampleIndexedMessages = TestUtils.readSampleData(sampleIndexedPath);
+ Assert.assertEquals(sampleIndexedMessages.size(), docs.size());
+ for (int i = 0; i < docs.size(); i++) {
+ String doc = docs.get(i).toString();
+ String sampleIndexedMessage = new String(sampleIndexedMessages.get(i));
+ assertEqual(sampleIndexedMessage, doc);
+ }
+ runner.stop();
+ }
+ public static void assertEqual(String doc1, String doc2) {
+ Assert.assertEquals(doc1.length(), doc2.length());
+ char[] c1 = doc1.toCharArray();
+ Arrays.sort(c1);
+ char[] c2 = doc2.toCharArray();
+ Arrays.sort(c2);
+ Assert.assertArrayEquals(c1, c2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/ParserIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/ParserIntegrationTest.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/ParserIntegrationTest.java
new file mode 100644
index 0000000..c55a069
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/ParserIntegrationTest.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration;
+
+import com.google.common.base.Function;
+import kafka.api.FetchRequest;
+import kafka.api.FetchRequestBuilder;
+import kafka.consumer.ConsumerIterator;
+import kafka.javaapi.FetchResponse;
+import kafka.javaapi.consumer.SimpleConsumer;
+import kafka.javaapi.producer.Producer;
+import kafka.message.MessageAndMetadata;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.metron.Constants;
+import org.apache.metron.integration.util.TestUtils;
+import org.apache.metron.integration.util.UnitTestHelper;
+import org.apache.metron.integration.util.integration.ComponentRunner;
+import org.apache.metron.integration.util.integration.Processor;
+import org.apache.metron.integration.util.integration.ReadinessState;
+import org.apache.metron.integration.util.integration.components.ElasticSearchComponent;
+import org.apache.metron.integration.util.integration.components.FluxTopologyComponent;
+import org.apache.metron.integration.util.integration.components.KafkaWithZKComponent;
+import org.apache.metron.integration.util.integration.util.KafkaUtil;
+import org.apache.metron.spout.pcap.HDFSWriterCallback;
+import org.apache.metron.test.converters.HexStringConverter;
+import org.apache.metron.utils.SourceConfigUtils;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.junit.Assert;
+import org.junit.Test;
+
+import javax.annotation.Nullable;
+import java.io.*;
+import java.util.*;
+
+public abstract class ParserIntegrationTest {
+
+ public abstract String getFluxPath();
+ public abstract String getSampleInputPath();
+ public abstract String getSampleParsedPath();
+ public abstract String getSourceType();
+ public abstract String getSourceConfig();
+ public abstract String getFluxTopicProperty();
+
+ @Test
+ public void test() throws Exception {
+
+ final String kafkaTopic = "test";
+
+ final List<byte[]> inputMessages = TestUtils.readSampleData(getSampleInputPath());
+
+ final Properties topologyProperties = new Properties() {{
+ setProperty(getFluxTopicProperty(), kafkaTopic);
+ }};
+ final KafkaWithZKComponent kafkaComponent = new KafkaWithZKComponent().withTopics(new ArrayList<KafkaWithZKComponent.Topic>() {{
+ add(new KafkaWithZKComponent.Topic(kafkaTopic, 1));
+ }})
+ .withPostStartCallback(new Function<KafkaWithZKComponent, Void>() {
+ @Nullable
+ @Override
+ public Void apply(@Nullable KafkaWithZKComponent kafkaWithZKComponent) {
+ topologyProperties.setProperty("kafka.zk", kafkaWithZKComponent.getZookeeperConnect());
+ try {
+ SourceConfigUtils.writeToZookeeper(getSourceType(), getSourceConfig().getBytes(), kafkaWithZKComponent.getZookeeperConnect());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+ });
+
+ topologyProperties.setProperty("kafka.broker", kafkaComponent.getBrokerList());
+ FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder()
+ .withTopologyLocation(new File(getFluxPath()))
+ .withTopologyName("test")
+ .withTopologyProperties(topologyProperties)
+ .build();
+
+ UnitTestHelper.verboseLogging();
+ ComponentRunner runner = new ComponentRunner.Builder()
+ .withComponent("kafka", kafkaComponent)
+ .withComponent("storm", fluxComponent)
+ .withTimeBetweenAttempts(5000)
+ .build();
+ runner.start();
+ fluxComponent.submitTopology();
+ kafkaComponent.writeMessages(kafkaTopic, inputMessages);
+ List<byte[]> outputMessages =
+ runner.process(new Processor<List<byte[]>>() {
+ List<byte[]> messages = null;
+
+ public ReadinessState process(ComponentRunner runner) {
+ KafkaWithZKComponent kafkaWithZKComponent = runner.getComponent("kafka", KafkaWithZKComponent.class);
+ List<byte[]> outputMessages = kafkaWithZKComponent.readMessages(Constants.ENRICHMENT_TOPIC);
+ if (outputMessages.size() == inputMessages.size()) {
+ messages = outputMessages;
+ return ReadinessState.READY;
+ } else {
+ return ReadinessState.NOT_READY;
+ }
+ }
+
+ public List<byte[]> getResult() {
+ return messages;
+ }
+ });
+ List<byte[]> sampleParsedMessages = TestUtils.readSampleData(getSampleParsedPath());
+ Assert.assertEquals(sampleParsedMessages.size(), outputMessages.size());
+ for (int i = 0; i < outputMessages.size(); i++) {
+ String sampleParsedMessage = new String(sampleParsedMessages.get(i));
+ String outputMessage = new String(outputMessages.get(i));
+ assertJSONEqual(sampleParsedMessage, outputMessage);
+ }
+ runner.stop();
+
+ }
+
+ public static void assertJSONEqual(String doc1, String doc2) throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ Map m1 = mapper.readValue(doc1, Map.class);
+ Map m2 = mapper.readValue(doc2, Map.class);
+ Assert.assertEquals(m1.size(), m2.size());
+ for(Object k : m1.keySet()) {
+ Object v1 = m1.get(k);
+ Object v2 = m2.get(k);
+
+ if(v2 == null) {
+ Assert.fail("Unable to find key: " + k + " in output");
+ }
+ if(k.equals("timestamp")) {
+ //TODO: Take the ?!?@ timestamps out of the reference file.
+ Assert.assertEquals(v1.toString().length(), v2.toString().length());
+ }
+ else if(!v2.equals(v1)) {
+ Assert.assertEquals("value mismatch for " + k ,v1, v2);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/SnortIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/SnortIntegrationTest.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/SnortIntegrationTest.java
new file mode 100644
index 0000000..7508ad7
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/SnortIntegrationTest.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration;
+
+public class SnortIntegrationTest extends ParserIntegrationTest {
+
+ @Override
+ public String getFluxPath() {
+ return "src/main/resources/Metron_Configs/topologies/snort/test.yaml";
+ }
+
+ @Override
+ public String getSampleInputPath() {
+ return "src/main/resources/SampleInput/SnortOutput";
+ }
+
+ @Override
+ public String getSampleParsedPath() {
+ return "src/main/resources/SampleParsed/SnortParsed";
+ }
+
+ @Override
+ public String getSourceType() {
+ return "snort";
+ }
+
+ @Override
+ public String getSourceConfig() {
+ return "{\"index\": \"snort\"," +
+ " \"batchSize\": 1," +
+ " \"enrichmentFieldMap\":" +
+ " {" +
+ " \"geo\": [\"src\", \"dst\"]," +
+ " \"host\": [\"src\", \"dst\"]" +
+ " }," +
+ " \"threatIntelFieldMap\":" +
+ " {" +
+ " \"ip\": [\"src\", \"dst\"]" +
+ " }" +
+ "}";
+ }
+
+ @Override
+ public String getFluxTopicProperty() {
+ return "spout.kafka.topic.snort";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/YafIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/YafIntegrationTest.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/YafIntegrationTest.java
new file mode 100644
index 0000000..cf91bea
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/YafIntegrationTest.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration;
+
+public class YafIntegrationTest extends ParserIntegrationTest {
+
+ @Override
+ public String getFluxPath() {
+ return "src/main/resources/Metron_Configs/topologies/yaf/test.yaml";
+ }
+
+ @Override
+ public String getSampleInputPath() {
+ return "src/main/resources/SampleInput/YafExampleOutput";
+ }
+
+ @Override
+ public String getSampleParsedPath() {
+ return "src/main/resources/SampleParsed/YafExampleParsed";
+ }
+
+ @Override
+ public String getSourceType() {
+ return "yaf";
+ }
+
+ @Override
+ public String getSourceConfig() {
+ return "{\"index\": \"yaf\"," +
+ " \"batchSize\": 5," +
+ " \"enrichmentFieldMap\":" +
+ " {" +
+ " \"geo\": [\"sip\", \"dip\"]," +
+ " \"host\": [\"sip\", \"dip\"]" +
+ " }," +
+ " \"threatIntelFieldMap\":" +
+ " {" +
+ " \"ip\": [\"sip\", \"dip\"]" +
+ " }" +
+ "}";
+ }
+
+ @Override
+ public String getFluxTopicProperty() {
+ return "spout.kafka.topic.yaf";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/pcap/PcapIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/pcap/PcapIntegrationTest.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/pcap/PcapIntegrationTest.java
deleted file mode 100644
index 3337855..0000000
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/pcap/PcapIntegrationTest.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.pcap;
-
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Iterables;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.metron.hbase.HTableProvider;
-import org.apache.metron.hbase.TableProvider;
-import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
-import org.apache.metron.integration.util.UnitTestHelper;
-import org.apache.metron.integration.util.integration.ComponentRunner;
-import org.apache.metron.integration.util.integration.Processor;
-import org.apache.metron.integration.util.integration.ReadinessState;
-import org.apache.metron.integration.util.integration.components.ElasticSearchComponent;
-import org.apache.metron.integration.util.integration.components.FluxTopologyComponent;
-import org.apache.metron.integration.util.mock.MockHTable;
-import org.apache.metron.integration.util.threatintel.ThreatIntelHelper;
-import org.apache.metron.parsing.parsers.PcapParser;
-import org.apache.metron.reference.lookup.LookupKV;
-import org.apache.metron.test.converters.HexStringConverter;
-import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
-import org.apache.metron.threatintel.ThreatIntelResults;
-import org.json.simple.JSONObject;
-import org.junit.Assert;
-import org.junit.Test;
-
-import javax.annotation.Nullable;
-import java.io.*;
-import java.text.SimpleDateFormat;
-import java.util.*;
-
-public class PcapIntegrationTest {
-
- private String topologiesDir = "src/main/resources/Metron_Configs/topologies";
- private String targetDir = "target";
-
- public static class Provider implements TableProvider, Serializable{
-
- MockHTable.Provider provider = new MockHTable.Provider();
- @Override
- public HTableInterface getTable(Configuration config, String tableName) throws IOException {
- return provider.getTable(config, tableName);
- }
- }
-
- @Test
- public void testTopology() throws Exception {
- if(!new File(topologiesDir).exists()) {
- topologiesDir = UnitTestHelper.findDir("topologies");
- }
- if(!new File(targetDir).exists()) {
- targetDir = UnitTestHelper.findDir("target");
- }
- Assert.assertNotNull(topologiesDir);
- Assert.assertNotNull(targetDir);
- final List<String> expectedPcapIds= getExpectedPcap(new File(topologiesDir + "/../../SampleInput/PCAPExampleOutput"));
- Assert.assertTrue("Expected non-zero number of PCAP Ids from the sample data", expectedPcapIds.size() > 0);
- System.out.println("Using topologies directory: " + topologiesDir);
-
- ElasticSearchComponent esComponent = new ElasticSearchComponent.Builder()
- .withHttpPort(9211)
- .withIndexDir(new File(targetDir + "/elasticsearch"))
- .build();
- final String cf = "cf";
- final String trackerHBaseTable = "tracker";
- final String ipThreatIntelTable = "ip_threat_intel";
- Properties topologyProperties = new Properties() {{
- setProperty("input.path", "src/main/resources/");
- setProperty("es.port", "9300");
- setProperty("es.ip", "localhost");
- setProperty("es.clustername", "metron");
- setProperty("mysql.ip", "node1");
- setProperty("mysql.port", "3306");
- setProperty("mysql.username", "root");
- setProperty("mysql.password", "P@ssw0rd");
- setProperty("pcap.binary.converter", "FROM_HEX_STRING");
- setProperty("testing.repeating", "false");
- setProperty("org.apache.metron.metrics.reporter.graphite", "false");
- setProperty("org.apache.metron.metrics.reporter.console", "false");
- setProperty("org.apache.metron.metrics.reporter.jmx", "false");
- setProperty("org.apache.metron.metrics.TelemetryParserBolt.acks","true");
- setProperty("org.apache.metron.metrics.TelemetryParserBolt.emits", "true");
- setProperty("org.apache.metron.metrics.TelemetryParserBolt.fails","true");
- setProperty("org.apache.metron.metrics.GenericEnrichmentBolt.acks","true");
- setProperty("org.apache.metron.metrics.GenericEnrichmentBolt.emits","true");
- setProperty("org.apache.metron.metrics.GenericEnrichmentBolt.fails","true");
- setProperty("org.apache.metron.metrics.TelemetryIndexingBolt.acks", "true");
- setProperty("org.apache.metron.metrics.TelemetryIndexingBolt.emits","true");
- setProperty("org.apache.metron.metrics.TelemetryIndexingBolt.fails","true");
- setProperty("kafka.zk", "localhost:2000,localhost:2000");
- setProperty("bolt.hbase.table.name", "pcap_test");
- setProperty("bolt.hbase.table.fields", "t:value");
- setProperty("bolt.hbase.table.key.tuple.field.name", "key");
- setProperty("bolt.hbase.table.timestamp.tuple.field.name", "timestamp");
- setProperty("bolt.hbase.enable.batching", "false");
- setProperty("bolt.hbase.write.buffer.size.in.bytes", "2000000");
- setProperty("bolt.hbase.durability", "SKIP_WAL");
- setProperty("bolt.hbase.partitioner.region.info.refresh.interval.mins","60");
- setProperty("hbase.provider.impl","" + Provider.class.getName());
- setProperty("threat.intel.tracker.table", trackerHBaseTable);
- setProperty("threat.intel.tracker.cf", cf);
- setProperty("threat.intel.ip.table", ipThreatIntelTable);
- setProperty("threat.intel.ip.cf", cf);
- setProperty("org.apache.metron.enrichment.host.known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"}," +
- "{\"ip\":\"10.1.128.237\", \"local\":\"UNKNOWN\", \"type\":\"unknown\", \"asset_value\" : \"important\"}," +
- "{\"ip\":\"10.60.10.254\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}," +
- "{\"ip\":\"10.0.2.15\", \"local\":\"YES\", " +
- "\"type\":\"printer\", \"asset_value\" : \"important\"}]");
- }};
- //create MockHBaseTables
- final MockHTable trackerTable = (MockHTable)MockHTable.Provider.addToCache(trackerHBaseTable, cf);
- final MockHTable ipTable = (MockHTable)MockHTable.Provider.addToCache(ipThreatIntelTable, cf);
- ThreatIntelHelper.INSTANCE.load(ipTable, cf, new ArrayList<LookupKV<ThreatIntelKey, ThreatIntelValue>>(){{
- add(new LookupKV<>(new ThreatIntelKey("10.0.2.3"), new ThreatIntelValue(new HashMap<String, String>())));
- }});
- final MockHTable pcapTable = (MockHTable) MockHTable.Provider.addToCache("pcap_test", "t");
- FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder()
- .withTopologyLocation(new File(topologiesDir + "/pcap/local.yaml"))
- .withTopologyName("pcap")
- .withTopologyProperties(topologyProperties)
- .build();
- //UnitTestHelper.verboseLogging();
- ComponentRunner runner = new ComponentRunner.Builder()
- .withComponent("elasticsearch", esComponent)
- .withComponent("storm", fluxComponent)
- .build();
-
- final String index = getIndex();
- System.out.println("Index of the run: " + index);
- runner.start();
- fluxComponent.submitTopology();
- List<Map<String, Object>> docs =
- runner.process(new Processor<List<Map<String, Object>>> () {
- List<Map<String, Object>> docs = null;
- public ReadinessState process(ComponentRunner runner){
- ElasticSearchComponent elasticSearchComponent = runner.getComponent("elasticsearch", ElasticSearchComponent.class);
- if(elasticSearchComponent.hasIndex(index)) {
- try {
- docs = elasticSearchComponent.getAllIndexedDocs(index);
- } catch (IOException e) {
- throw new IllegalStateException("Unable to retrieve indexed documents.", e);
- }
- if(docs.size() < expectedPcapIds.size() && pcapTable.getPutLog().size() < expectedPcapIds.size()) {
- return ReadinessState.NOT_READY;
- }
- else {
- return ReadinessState.READY;
- }
- }
- else {
- return ReadinessState.NOT_READY;
- }
- }
-
- public List<Map<String, Object>> getResult() {
- return docs;
- }
- });
-
- Assert.assertEquals(expectedPcapIds.size(), pcapTable.getPutLog().size());
- UnitTestHelper.assertSetEqual("PCap IDs from Index"
- , new HashSet<>(expectedPcapIds)
- , convertToSet(Iterables.transform(docs, DOC_TO_PCAP_ID))
- );
- UnitTestHelper.assertSetEqual("PCap IDs from HBase"
- , new HashSet<>(expectedPcapIds)
- , convertToSet(Iterables.transform(pcapTable.getPutLog(), RK_TO_PCAP_ID))
- );
- Iterable<JSONObject> packetsFromHBase = Iterables.transform(pcapTable.getPutLog(), PUT_TO_PCAP);
- Assert.assertEquals(expectedPcapIds.size(), Iterables.size(packetsFromHBase));
-
- List<Map<String, Object>> allDocs= runner.getComponent("elasticsearch", ElasticSearchComponent.class).getAllIndexedDocs(index, null);
- boolean hasThreat = false;
- for(Map<String, Object> d : allDocs) {
- Map<String, Object> message = (Map<String, Object>) d.get("message");
- Set<String> ips = new HashSet<>(Arrays.asList((String)message.get("ip_dst_addr"), (String)message.get("ip_src_addr")));
- if(ips.contains("10.0.2.3")) {
- hasThreat = true;
- Map<String, Object> alerts = (Map<String, Object>) ((Map<String, Object>) d.get("alerts")).get("ip");
- Assert.assertTrue( ((Map<String,Object>)alerts.get("ip_dst_addr")).size() > 0
- || ((Map<String,Object>)alerts.get("ip_src_addr")).size() > 0
- );
- }
- }
- Assert.assertTrue(hasThreat);
- MockHTable.Provider.clear();
- runner.stop();
- }
-
- public static Set<String> convertToSet(Iterable<String> strings) {
- Set<String> ret = new HashSet<String>();
- Iterables.addAll(ret, strings);
- return ret;
- }
- public static final Function<Put, String> RK_TO_PCAP_ID = new Function<Put, String>() {
- @Nullable
- public String apply(@Nullable Put put) {
- String rk =new String(put.getRow());
- return Joiner.on("-").join(Iterables.limit(Splitter.on('-').split(rk), 5));
- }
- };
-
- public static final Function<Map<String, Object>, String> DOC_TO_PCAP_ID = new Function<Map<String, Object>, String>() {
-
- @Nullable
- public String apply(@Nullable Map<String, Object> doc) {
- return (String)doc.get("pcap_id");
- }
- };
-
- public static final Function<Put, JSONObject> PUT_TO_PCAP = new
- Function<Put, JSONObject>() {
- @Nullable
- public JSONObject apply(@Nullable Put put) {
- try {
- return putToPcap(put);
- } catch (IOException e) {
- throw new RuntimeException("Unable to convert put to PCAP: " + put);
- }
- }
- };
-
-
-
- private static List<String> getExpectedPcap(File rawFile) throws IOException {
- List<String> ret = new ArrayList<String>();
- PcapParser parser = new PcapParser();
- parser.withTsPrecision("MICRO");
- parser.init();
- BufferedReader br = new BufferedReader(new FileReader(rawFile));
- for(String line = null; (line = br.readLine()) != null;) {
- byte[] pcapBytes = new HexStringConverter().convert(line);
- List<JSONObject> list = parser.parse(pcapBytes);
- for(JSONObject message : list) {
- ret.add((String) message.get("pcap_id"));
- }
- }
- return ret;
- }
-
- private static String getIndex() {
- SimpleDateFormat sdf = new SimpleDateFormat("yyyy.MM.dd.hh");
- Date d = new Date();
- return "pcap_index_" + sdf.format(d);
- }
-
- private static JSONObject putToPcap(Put p) throws IOException {
- PcapParser parser = new PcapParser();
- parser.init();
- List<Cell> cells = p.get(Bytes.toBytes("t"), Bytes.toBytes("value"));
- Assert.assertEquals(1, cells.size());
- List<JSONObject> messages = parser.parse(cells.get(0).getValueArray());
- Assert.assertEquals(1, messages.size());
- return messages.get(0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/TestUtils.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/TestUtils.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/TestUtils.java
new file mode 100644
index 0000000..594700b
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/TestUtils.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration.util;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class TestUtils {
+
+ public static List<byte[]> readSampleData(String samplePath) throws IOException {
+ BufferedReader br = new BufferedReader(new FileReader(samplePath));
+ List<byte[]> ret = new ArrayList<>();
+ for (String line = null; (line = br.readLine()) != null; ) {
+ long ts = System.currentTimeMillis();
+ ret.add(line.getBytes());
+ }
+ br.close();
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a4c773d..499e323 100644
--- a/pom.xml
+++ b/pom.xml
@@ -51,7 +51,7 @@
<exclude>metron-ui/lib/public/**</exclude>
<exclude>**/src/main/resources/patterns/**</exclude>
<exclude>**/src/test/resources/**</exclude>
- <exclude>**/src/main/resources/SampleInput/**</exclude>
+ <exclude>**/src/main/resources/Sample*/**</exclude>
<exclude>**/dependency-reduced-pom.xml</exclude>
<exclude>**/files/opensoc-ui</exclude>
<exclude>**/*.iml</exclude>
[3/9] incubator-metron git commit: METRON-56 Create unified
enrichment topology (merrimanr via cestella) closes
apache/incubator-metron#33
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/remote.yaml
index 957677b..5bc5f76 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/remote.yaml
@@ -21,143 +21,10 @@ config:
components:
- id: "parser"
className: "org.apache.metron.parsing.parsers.BasicSourcefireParser"
- - id: "jdbcConfig"
- className: "org.apache.metron.enrichment.adapters.jdbc.MySqlConfig"
- properties:
- - name: "host"
- value: "${mysql.ip}"
- - name: "port"
- value: ${mysql.port}
- - name: "username"
- value: "${mysql.username}"
- - name: "password"
- value: "${mysql.password}"
- - name: "table"
- value: "GEO"
- - id: "geoEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
- configMethods:
- - name: "withJdbcConfig"
- args:
- - ref: "jdbcConfig"
- - id: "geoEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "geo"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "geoEnrichmentAdapter"
- - id: "hostEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
constructorArgs:
- - '${org.apache.metron.enrichment.host.known_hosts}'
- - id: "hostEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "host"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "hostEnrichmentAdapter"
- - id: "enrichments"
- className: "java.util.ArrayList"
- configMethods:
- - name: "add"
- args:
- - ref: "geoEnrichment"
- - name: "add"
- args:
- - ref: "hostEnrichment"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "alertsConfig"
- className: "java.util.HashMap"
- configMethods:
- - name: "put"
- args: ["whitelist_table_name", "ip_whitelist"]
- - name: "put"
- args: ["blacklist_table_name", "ip_blacklist"]
- - name: "put"
- args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"]
- - name: "put"
- args: ["port", "2181"]
- - name: "put"
- args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"]
- - name: "put"
- args: ["_MAX_TIME_RETAIN_MINUTES", "1000"]
- - id: "alertsAdapter"
- className: "org.apache.metron.alerts.adapters.CIFAlertsAdapter"
- constructorArgs:
- - ref: "alertsConfig"
- - id: "alertsIdentifier"
- className: "org.json.simple.JSONObject"
- configMethods:
- - name: "put"
- args: ["environment", "local"]
- - name: "put"
- args: ["topology", "sourcefire"]
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+ - "${kafka.broker}"
- id: "zkHosts"
className: "storm.kafka.ZkHosts"
constructorArgs:
@@ -168,18 +35,28 @@ components:
# zookeeper hosts
- ref: "zkHosts"
# topic name
- - "${spout.kafka.topic.pcap}"
+ - "${spout.kafka.topic.snort}"
# zk root
- ""
# id
- - "${spout.kafka.topic.pcap}"
+ - "${spout.kafka.topic.snort}"
properties:
- - name: "forceFromStart"
+ - name: "ignoreZkOffsets"
value: true
- name: "startOffsetTime"
value: -1
spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - true
- id: "kafkaSpout"
className: "storm.kafka.KafkaSpout"
constructorArgs:
@@ -187,229 +64,16 @@ spouts:
bolts:
- id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "parser"
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "sourcefire_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "sourcefire_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsBolt"
- className: "org.apache.metron.alerts.TelemetryAlertsBolt"
- configMethods:
- - name: "withIdentifier"
- args:
- - ref: "alertsIdentifier"
- - name: "withMaxCacheSize"
- args: [1000]
- - name: "withMaxTimeRetain"
- args: [3600]
- - name: "withAlertsAdapter"
- args:
- - ref: "alertsAdapter"
- - name: "withOutputFieldName"
- args: ["message"]
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "alert"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.ww"
- - name: "withDocumentName"
- args:
- - "sourcefire_alert"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "sourcefire_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "geoEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "geoEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "hostEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "hostEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "joinBolt"
- className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt"
- configMethods:
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "${spout.kafka.topic.snort}"
+ - ref: "parser"
+ - ref: "writer"
streams:
- - name: "spout -> parser"
+ - name: "spout -> bolt"
from: "kafkaSpout"
to: "parserBolt"
grouping:
type: SHUFFLE
- - name: "parser -> host"
- from: "parserBolt"
- to: "hostEnrichmentBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "parser -> geo"
- from: "parserBolt"
- to: "geoEnrichmentBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "parser -> join"
- from: "parserBolt"
- to: "joinBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "geo -> join"
- from: "geoEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "host -> join"
- from: "hostEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "join -> alerts"
- from: "joinBolt"
- to: "alertsBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "alerts -> alertsIndexing"
- from: "alertsBolt"
- to: "alertsIndexingBolt"
- grouping:
- streamId: "message"
- type: SHUFFLE
- - name: "join -> indexing"
- from: "joinBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "alerts -> errors"
- from: "alertsBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/test.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/test.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/test.yaml
new file mode 100644
index 0000000..e9e583a
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/test.yaml
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "sourcefire-test"
+config:
+ topology.workers: 1
+
+components:
+ - id: "parser"
+ className: "org.apache.metron.parsing.parsers.BasicSourcefireParser"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
+ constructorArgs:
+ - "${kafka.broker}"
+ - id: "zkHosts"
+ className: "storm.kafka.ZkHosts"
+ constructorArgs:
+ - "${kafka.zk}"
+ - id: "kafkaConfig"
+ className: "storm.kafka.SpoutConfig"
+ constructorArgs:
+ # zookeeper hosts
+ - ref: "zkHosts"
+ # topic name
+ - "${spout.kafka.topic.snort}"
+ # zk root
+ - ""
+ # id
+ - "${spout.kafka.topic.snort}"
+ properties:
+ - name: "ignoreZkOffsets"
+ value: true
+ - name: "startOffsetTime"
+ value: -2
+
+spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - false
+ - id: "kafkaSpout"
+ className: "storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+
+bolts:
+ - id: "parserBolt"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "${spout.kafka.topic.snort}"
+ - ref: "parser"
+ - ref: "writer"
+
+streams:
+ - name: "spout -> bolt"
+ from: "kafkaSpout"
+ to: "parserBolt"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/local.yaml
deleted file mode 100644
index cf026a2..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/local.yaml
+++ /dev/null
@@ -1,192 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "yaf-local"
-config:
- topology.workers: 1
-
-components:
- - id: "yafParser"
- className: "org.apache.metron.parsing.parsers.BasicYafParser"
- - id: "genericMessageFilter"
- className: "org.apache.metron.filters.GenericMessageFilter"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
-
-spouts:
- - id: "testingSpout"
- className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
- parallelism: 1
- configMethods:
- - name: "withFilename"
- args:
- - "SampleInput/YafExampleOutput"
- - name: "withRepeating"
- args:
- - true
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "yafParser"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "yaf_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "yaf_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "yaf_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
-
-streams:
- - name: "spout -> parser"
- from: "testingSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
- - name: "parser -> indexing"
- from: "parserBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/remote.yaml
index 65cff0f..98395e9 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/remote.yaml
@@ -14,76 +14,31 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: "yaf"
+name: "yaf-test"
config:
topology.workers: 1
+
components:
- - id: "yafParser"
- className: "org.apache.metron.parsing.parsers.BasicYafParser"
- - id: "genericMessageFilter"
- className: "org.apache.metron.filters.GenericMessageFilter"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
+ - id: "parser"
+ className: "org.apache.metron.parsing.parsers.GrokParser"
+ constructorArgs:
+ - "/patterns/yaf"
+ - "YAF_DELIMITED"
configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
+ - name: "withTimestampField"
args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
+ - "start_time"
+ - name: "withTimeFields"
args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
+ - ["start_time", "end_time"]
+ - name: "withDateFormat"
args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+ - "yyyy-MM-dd HH:mm:ss.S"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
+ constructorArgs:
+ - "${kafka.broker}"
- id: "zkHosts"
className: "storm.kafka.ZkHosts"
constructorArgs:
@@ -100,12 +55,24 @@ components:
# id
- "${spout.kafka.topic.yaf}"
properties:
- - name: "forceFromStart"
+ - name: "ignoreZkOffsets"
value: true
- name: "startOffsetTime"
value: -1
+ - name: "socketTimeoutMs"
+ value: 1000000
spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - true
- id: "kafkaSpout"
className: "storm.kafka.KafkaSpout"
constructorArgs:
@@ -113,94 +80,16 @@ spouts:
bolts:
- id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "yafParser"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "yaf_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "yaf_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "yaf_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "yaf"
+ - ref: "parser"
+ - ref: "writer"
streams:
- - name: "spout -> parser"
+ - name: "spout -> bolt"
from: "kafkaSpout"
to: "parserBolt"
grouping:
type: SHUFFLE
- - name: "parser -> indexing"
- from: "parserBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/test.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/test.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/test.yaml
new file mode 100644
index 0000000..021d3f8
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/yaf/test.yaml
@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "yaf-test"
+config:
+ topology.workers: 1
+
+
+components:
+ - id: "parser"
+ className: "org.apache.metron.parsing.parsers.GrokParser"
+ constructorArgs:
+ - "../Metron-MessageParsers/src/main/resources/patterns/yaf"
+ - "YAF_DELIMITED"
+ configMethods:
+ - name: "withTimestampField"
+ args:
+ - "start_time"
+ - name: "withTimeFields"
+ args:
+ - ["start_time", "end_time"]
+ - name: "withDateFormat"
+ args:
+ - "yyyy-MM-dd HH:mm:ss.S"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
+ constructorArgs:
+ - "${kafka.broker}"
+ - id: "zkHosts"
+ className: "storm.kafka.ZkHosts"
+ constructorArgs:
+ - "${kafka.zk}"
+ - id: "kafkaConfig"
+ className: "storm.kafka.SpoutConfig"
+ constructorArgs:
+ # zookeeper hosts
+ - ref: "zkHosts"
+ # topic name
+ - "${spout.kafka.topic.yaf}"
+ # zk root
+ - ""
+ # id
+ - "${spout.kafka.topic.yaf}"
+ properties:
+ - name: "ignoreZkOffsets"
+ value: true
+ - name: "startOffsetTime"
+ value: -2
+ - name: "socketTimeoutMs"
+ value: 1000000
+
+spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - false
+ - id: "kafkaSpout"
+ className: "storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+
+bolts:
+ - id: "parserBolt"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "yaf"
+ - ref: "parser"
+ - ref: "writer"
+
+streams:
+ - name: "spout -> bolt"
+ from: "kafkaSpout"
+ to: "parserBolt"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/SampleIndexed/YafIndexed
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/SampleIndexed/YafIndexed b/metron-streaming/Metron-Topologies/src/main/resources/SampleIndexed/YafIndexed
new file mode 100644
index 0000000..27b3589
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/SampleIndexed/YafIndexed
@@ -0,0 +1,10 @@
+{enrichments.geo.dip.longitude=test longitude, iflags=AS, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=22efa001, dip=10.0.2.15, dp=39468, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=216.21.170.221, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp
=80, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988512, app=0, threatintels.ip.sip=, oct=44, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988512, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988512, riflags=0, rtt=0.000, proto=6, enrichments.host.dip.known_info.local=YES}
+{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=10000000, dip=10.0.2.3, enrichments.host.sip.known_info.local=YES, dp=53, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longit
ude,test latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=37299, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988502, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=56, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, end_time=1453994988502, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988502, riflags=0, rtt=0.000, threatintels.ip.dip.threat_source=ip_threat_intel, proto=17}
+{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=0, dip=10.0.2.15, dp=37299, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.3, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=53, enrichmen
ts.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988504, app=0, oct=312, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988504, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988504, threatintels.ip.sip.threat_source=ip_threat_intel, riflags=0, rtt=0.000, proto=17, enrichments.host.dip.known_info.local=YES}
+{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=0, dip=10.0.2.3, enrichments.host.sip.known_info.local=YES, dp=53, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,tes
t latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=56303, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988504, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=56, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, end_time=1453994988504, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988504, riflags=0, rtt=0.000, threatintels.ip.dip.threat_source=ip_threat_intel, proto=17}
+{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=0, dip=10.0.2.15, dp=56303, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.3, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=53, enrichmen
ts.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988506, app=0, oct=84, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988506, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988506, threatintels.ip.sip.threat_source=ip_threat_intel, riflags=0, rtt=0.000, proto=17, enrichments.host.dip.known_info.local=YES}
+{enrichments.geo.dip.longitude=test longitude, iflags=S, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=58c52fca, dip=216.21.170.221, enrichments.host.sip.known_info.local=YES, dp=80, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.si
p.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=39468, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988508, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=60, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, end_time=1453994988508, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988508, riflags=0, rtt=0.000, proto=6}
+{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=58c52fcb, dip=216.21.170.221, enrichments.host.sip.known_info.local=YES, dp=80, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle , enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.s
ip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=39468, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988512, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=40, end_reason=idle , enrichments.geo.sip.locID=1, risn=0, end_time=1453994988512, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988512, riflags=0, rtt=0.000, proto=6}
+{enrichments.geo.dip.longitude=test longitude, iflags=AP, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=58c52fcb, dip=216.21.170.221, enrichments.host.sip.known_info.local=YES, dp=80, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle , enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.
sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=39468, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988512, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=148, end_reason=idle , enrichments.geo.sip.locID=1, risn=0, end_time=1453994988512, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988512, riflags=0, rtt=0.000, proto=6}
+{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=22efa002, dip=10.0.2.15, dp=39468, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle , enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=216.21.170.221, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp
=80, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988512, app=0, threatintels.ip.sip=, oct=40, end_reason=idle , enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988512, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988512, riflags=0, rtt=0.000, proto=6, enrichments.host.dip.known_info.local=YES}
+{enrichments.geo.dip.longitude=test longitude, iflags=AP, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=22efa002, dip=10.0.2.15, dp=39468, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=216.21.170.221, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp
=80, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988562, app=0, threatintels.ip.sip=, oct=604, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988562, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988562, riflags=0, rtt=0.000, proto=6, enrichments.host.dip.known_info.local=YES}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/SnortOutput
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/SnortOutput b/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/SnortOutput
index 2b9836e..0497b0f 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/SnortOutput
+++ b/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/SnortOutput
@@ -1 +1,3 @@
-01/27-16:01:04.877970 ,129,12,1,"Consecutive TCP small segments exceeding threshold",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,,
\ No newline at end of file
+01/27-16:01:04.877970 ,129,12,1,"Consecutive TCP small segments exceeding threshold",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,,
+02/22-15:56:48.612494 ,129,12,1,"Consecutive TCP small segments exceeding threshold",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0x6E,***AP***,0xDB45F7A,0x7701DD5B,,0xFFFF,64,0,16785,96,98304,,,,
+02/22-15:56:48.616775 ,129,12,1,"Consecutive TCP small segments exceeding threshold",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0xA6,***AP***,0xDB508F2,0x7701DD5B,,0xFFFF,64,0,16824,152,155648,,,,
\ No newline at end of file
[9/9] incubator-metron git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-metron
Posted by ce...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-metron
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/3be012db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/3be012db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/3be012db
Branch: refs/heads/master
Commit: 3be012db93c0ac393078462b695b4cb5bd40728d
Parents: 9f96399 a7e3879
Author: cstella <ce...@gmail.com>
Authored: Tue Mar 1 13:20:20 2016 -0500
Committer: cstella <ce...@gmail.com>
Committed: Tue Mar 1 13:20:20 2016 -0500
----------------------------------------------------------------------
deployment/roles/pcap_replay/README.md | 27 ++++--
deployment/roles/pcap_replay/files/pcap-replay | 90 -------------------
deployment/roles/pcap_replay/meta/main.yml | 34 ++++++++
deployment/roles/pcap_replay/tasks/main.yml | 4 +-
deployment/roles/pcap_replay/tasks/service.yml | 14 +--
.../roles/pcap_replay/tasks/tcpreplay.yml | 4 +-
.../roles/pcap_replay/templates/pcap-replay | 92 ++++++++++++++++++++
deployment/roles/pcap_replay/vars/main.yml | 4 +-
8 files changed, 159 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
[5/9] incubator-metron git commit: METRON-56 Create unified
enrichment topology (merrimanr via cestella) closes
apache/incubator-metron#33
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/enrichment/test.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/enrichment/test.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/enrichment/test.yaml
new file mode 100644
index 0000000..0e530f5
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/enrichment/test.yaml
@@ -0,0 +1,314 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "enrichment"
+config:
+ topology.workers: 1
+
+components:
+# Enrichment
+ - id: "geoEnrichmentAdapter"
+ className: "org.apache.metron.integration.util.mock.MockGeoAdapter"
+ - id: "geoEnrichment"
+ className: "org.apache.metron.domain.Enrichment"
+ constructorArgs:
+ - "geo"
+ - ref: "geoEnrichmentAdapter"
+ - id: "hostEnrichmentAdapter"
+ className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
+ constructorArgs:
+ - '${org.apache.metron.enrichment.host.known_hosts}'
+ - id: "hostEnrichment"
+ className: "org.apache.metron.domain.Enrichment"
+ constructorArgs:
+ - "host"
+ - ref: "hostEnrichmentAdapter"
+ - id: "enrichments"
+ className: "java.util.ArrayList"
+ configMethods:
+ - name: "add"
+ args:
+ - ref: "geoEnrichment"
+ - name: "add"
+ args:
+ - ref: "hostEnrichment"
+
+# Threat Intel
+ - id: "ipThreatIntelConfig"
+ className: "org.apache.metron.threatintel.ThreatIntelConfig"
+ configMethods:
+ - name: "withProviderImpl"
+ args:
+ - "${hbase.provider.impl}"
+ - name: "withTrackerHBaseTable"
+ args:
+ - "${threat.intel.tracker.table}"
+ - name: "withTrackerHBaseCF"
+ args:
+ - "${threat.intel.tracker.cf}"
+ - name: "withHBaseTable"
+ args:
+ - "${threat.intel.ip.table}"
+ - name: "withHBaseCF"
+ args:
+ - "${threat.intel.ip.cf}"
+ - id: "ipThreatIntelAdapter"
+ className: "org.apache.metron.threatintel.ThreatIntelAdapter"
+ configMethods:
+ - name: "withConfig"
+ args:
+ - ref: "ipThreatIntelConfig"
+ - id: "ipThreatIntelEnrichment"
+ className: "org.apache.metron.domain.Enrichment"
+ constructorArgs:
+ - "ip"
+ - ref: "ipThreatIntelAdapter"
+ - id: "threatIntels"
+ className: "java.util.ArrayList"
+ configMethods:
+ - name: "add"
+ args:
+ - ref: "ipThreatIntelEnrichment"
+
+#indexing
+ - id: "indexWriter"
+ className: "org.apache.metron.writer.ElasticsearchWriter"
+ constructorArgs:
+ - "${es.clustername}"
+ - "${es.ip}"
+ - ${es.port}
+ - "${index.date.format}"
+
+#kafka/zookeeper
+ - id: "zkHosts"
+ className: "storm.kafka.ZkHosts"
+ constructorArgs:
+ - "${kafka.zk}"
+ - id: "kafkaConfig"
+ className: "storm.kafka.SpoutConfig"
+ constructorArgs:
+ # zookeeper hosts
+ - ref: "zkHosts"
+ # topic name
+ - "enrichments"
+ # zk root
+ - ""
+ # id
+ - "enrichments"
+ properties:
+ - name: "ignoreZkOffsets"
+ value: true
+ - name: "startOffsetTime"
+ value: -2
+
+spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - true
+ - id: "kafkaSpout"
+ className: "storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+bolts:
+# Enrichment Bolts
+ - id: "enrichmentSplitBolt"
+ className: "org.apache.metron.enrichment.bolt.EnrichmentSplitterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichments"
+ args:
+ - ref: "enrichments"
+ - id: "geoEnrichmentBolt"
+ className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichment"
+ args:
+ - ref: "geoEnrichment"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+ - id: "hostEnrichmentBolt"
+ className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichment"
+ args:
+ - ref: "hostEnrichment"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+ - id: "enrichmentJoinBolt"
+ className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichments"
+ args:
+ - ref: "enrichments"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+
+# Threat Intel Bolts
+ - id: "threatIntelSplitBolt"
+ className: "org.apache.metron.enrichment.bolt.ThreatIntelSplitterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichments"
+ args:
+ - ref: "threatIntels"
+ - name: "withMessageFieldName"
+ args: ["message"]
+ - id: "ipThreatIntelBolt"
+ className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichment"
+ args:
+ - ref: "ipThreatIntelEnrichment"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+ - id: "threatIntelJoinBolt"
+ className: "org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichments"
+ args:
+ - ref: "threatIntels"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+# Indexing Bolts
+ - id: "indexingBolt"
+ className: "org.apache.metron.bolt.BulkMessageWriterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withBulkMessageWriter"
+ args:
+ - ref: "indexWriter"
+
+
+streams:
+#parser
+ - name: "spout -> enrichmentSplit"
+ from: "kafkaSpout"
+ to: "enrichmentSplitBolt"
+ grouping:
+ type: SHUFFLE
+
+#enrichment
+ - name: "enrichmentSplit -> host"
+ from: "enrichmentSplitBolt"
+ to: "hostEnrichmentBolt"
+ grouping:
+ streamId: "host"
+ type: FIELDS
+ args: ["key"]
+ - name: "enrichmentSplit -> geo"
+ from: "enrichmentSplitBolt"
+ to: "geoEnrichmentBolt"
+ grouping:
+ streamId: "geo"
+ type: FIELDS
+ args: ["key"]
+ - name: "splitter -> join"
+ from: "enrichmentSplitBolt"
+ to: "enrichmentJoinBolt"
+ grouping:
+ streamId: "message"
+ type: FIELDS
+ args: ["key"]
+ - name: "geo -> join"
+ from: "geoEnrichmentBolt"
+ to: "enrichmentJoinBolt"
+ grouping:
+ streamId: "geo"
+ type: FIELDS
+ args: ["key"]
+ - name: "host -> join"
+ from: "hostEnrichmentBolt"
+ to: "enrichmentJoinBolt"
+ grouping:
+ streamId: "host"
+ type: FIELDS
+ args: ["key"]
+
+#threat intel
+ - name: "enrichmentJoin -> threatSplit"
+ from: "enrichmentJoinBolt"
+ to: "threatIntelSplitBolt"
+ grouping:
+ streamId: "message"
+ type: FIELDS
+ args: ["key"]
+
+ - name: "threatSplit -> ip"
+ from: "threatIntelSplitBolt"
+ to: "ipThreatIntelBolt"
+ grouping:
+ streamId: "ip"
+ type: FIELDS
+ args: ["key"]
+
+ - name: "ip -> join"
+ from: "ipThreatIntelBolt"
+ to: "threatIntelJoinBolt"
+ grouping:
+ streamId: "ip"
+ type: FIELDS
+ args: ["key"]
+ - name: "threatIntelSplit -> threatIntelJoin"
+ from: "threatIntelSplitBolt"
+ to: "threatIntelJoinBolt"
+ grouping:
+ streamId: "message"
+ type: FIELDS
+ args: ["key"]
+#indexing
+ - name: "threatIntelJoin -> indexing"
+ from: "threatIntelJoinBolt"
+ to: "indexingBolt"
+ grouping:
+ streamId: "message"
+ type: FIELDS
+ args: ["key"]
+ - name: "indexingBolt -> errorIndexingBolt"
+ from: "indexingBolt"
+ to: "indexingBolt"
+ grouping:
+ streamId: "error"
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/local.yaml
deleted file mode 100644
index 9a3c471..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/local.yaml
+++ /dev/null
@@ -1,401 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "fireeye-local"
-config:
- topology.workers: 1
-
-components:
- - id: "parser"
- className: "org.apache.metron.parsing.parsers.BasicFireEyeParser"
- - id: "jdbcConfig"
- className: "org.apache.metron.enrichment.adapters.jdbc.MySqlConfig"
- properties:
- - name: "host"
- value: "${mysql.ip}"
- - name: "port"
- value: ${mysql.port}
- - name: "username"
- value: "${mysql.username}"
- - name: "password"
- value: "${mysql.password}"
- - name: "table"
- value: "GEO"
- - id: "geoEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
- configMethods:
- - name: "withJdbcConfig"
- args:
- - ref: "jdbcConfig"
- - id: "geoEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "geo"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "geoEnrichmentAdapter"
- - id: "hostEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
- constructorArgs:
- - '${org.apache.metron.enrichment.host.known_hosts}'
- - id: "hostEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "host"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "hostEnrichmentAdapter"
- - id: "enrichments"
- className: "java.util.ArrayList"
- configMethods:
- - name: "add"
- args:
- - ref: "geoEnrichment"
- - name: "add"
- args:
- - ref: "hostEnrichment"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "alertsConfig"
- className: "java.util.HashMap"
- configMethods:
- - name: "put"
- args: ["whitelist_table_name", "ip_whitelist"]
- - name: "put"
- args: ["blacklist_table_name", "ip_blacklist"]
- - name: "put"
- args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"]
- - name: "put"
- args: ["port", "2181"]
- - name: "put"
- args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"]
- - name: "put"
- args: ["_MAX_TIME_RETAIN_MINUTES", "1000"]
- - id: "alertsAdapter"
- className: "org.apache.metron.alerts.adapters.CIFAlertsAdapter"
- constructorArgs:
- - ref: "alertsConfig"
- - id: "alertsIdentifier"
- className: "org.json.simple.JSONObject"
- configMethods:
- - name: "put"
- args: ["environment", "local"]
- - name: "put"
- args: ["topology", "fireeye"]
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
-
-spouts:
- - id: "testingSpout"
- className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
- parallelism: 1
- configMethods:
- - name: "withFilename"
- args:
- - "SampleInput/FireeyeExampleOutput"
- - name: "withRepeating"
- args:
- - true
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "parser"
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "fireeye_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "fireeye_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsBolt"
- className: "org.apache.metron.alerts.TelemetryAlertsBolt"
- configMethods:
- - name: "withIdentifier"
- args:
- - ref: "alertsIdentifier"
- - name: "withMaxCacheSize"
- args: [1000]
- - name: "withMaxTimeRetain"
- args: [3600]
- - name: "withAlertsAdapter"
- args:
- - ref: "alertsAdapter"
- - name: "withOutputFieldName"
- args: ["message"]
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "alert"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.ww"
- - name: "withDocumentName"
- args:
- - "fireeye_alert"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "fireeye_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "geoEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "geoEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "hostEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "hostEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "joinBolt"
- className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt"
- configMethods:
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
-
-streams:
- - name: "spout -> parser"
- from: "testingSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
- - name: "parser -> host"
- from: "parserBolt"
- to: "hostEnrichmentBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "parser -> geo"
- from: "parserBolt"
- to: "geoEnrichmentBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "parser -> join"
- from: "parserBolt"
- to: "joinBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "geo -> join"
- from: "geoEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "host -> join"
- from: "hostEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "join -> alerts"
- from: "joinBolt"
- to: "alertsBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "alerts -> alertsIndexing"
- from: "alertsBolt"
- to: "alertsIndexingBolt"
- grouping:
- streamId: "message"
- type: SHUFFLE
- - name: "join -> indexing"
- from: "joinBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "alerts -> errors"
- from: "alertsBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/remote.yaml
index cea5990..59cc372 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/remote.yaml
@@ -21,143 +21,10 @@ config:
components:
- id: "parser"
className: "org.apache.metron.parsing.parsers.BasicFireEyeParser"
- - id: "jdbcConfig"
- className: "org.apache.metron.enrichment.adapters.jdbc.MySqlConfig"
- properties:
- - name: "host"
- value: "${mysql.ip}"
- - name: "port"
- value: ${mysql.port}
- - name: "username"
- value: "${mysql.username}"
- - name: "password"
- value: "${mysql.password}"
- - name: "table"
- value: "GEO"
- - id: "geoEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
- configMethods:
- - name: "withJdbcConfig"
- args:
- - ref: "jdbcConfig"
- - id: "geoEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "geo"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "geoEnrichmentAdapter"
- - id: "hostEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
constructorArgs:
- - '${org.apache.metron.enrichment.host.known_hosts}'
- - id: "hostEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "host"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "hostEnrichmentAdapter"
- - id: "enrichments"
- className: "java.util.ArrayList"
- configMethods:
- - name: "add"
- args:
- - ref: "geoEnrichment"
- - name: "add"
- args:
- - ref: "hostEnrichment"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "alertsConfig"
- className: "java.util.HashMap"
- configMethods:
- - name: "put"
- args: ["whitelist_table_name", "ip_whitelist"]
- - name: "put"
- args: ["blacklist_table_name", "ip_blacklist"]
- - name: "put"
- args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"]
- - name: "put"
- args: ["port", "2181"]
- - name: "put"
- args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"]
- - name: "put"
- args: ["_MAX_TIME_RETAIN_MINUTES", "1000"]
- - id: "alertsAdapter"
- className: "org.apache.metron.alerts.adapters.CIFAlertsAdapter"
- constructorArgs:
- - ref: "alertsConfig"
- - id: "alertsIdentifier"
- className: "org.json.simple.JSONObject"
- configMethods:
- - name: "put"
- args: ["environment", "local"]
- - name: "put"
- args: ["topology", "fireeye"]
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+ - "${kafka.broker}"
- id: "zkHosts"
className: "storm.kafka.ZkHosts"
constructorArgs:
@@ -168,18 +35,28 @@ components:
# zookeeper hosts
- ref: "zkHosts"
# topic name
- - "${spout.kafka.topic.pcap}"
+ - "${spout.kafka.topic.snort}"
# zk root
- ""
# id
- - "${spout.kafka.topic.pcap}"
+ - "${spout.kafka.topic.snort}"
properties:
- - name: "forceFromStart"
+ - name: "ignoreZkOffsets"
value: true
- name: "startOffsetTime"
value: -1
spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - true
- id: "kafkaSpout"
className: "storm.kafka.KafkaSpout"
constructorArgs:
@@ -187,229 +64,16 @@ spouts:
bolts:
- id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "parser"
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "fireeye_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "fireeye_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsBolt"
- className: "org.apache.metron.alerts.TelemetryAlertsBolt"
- configMethods:
- - name: "withIdentifier"
- args:
- - ref: "alertsIdentifier"
- - name: "withMaxCacheSize"
- args: [1000]
- - name: "withMaxTimeRetain"
- args: [3600]
- - name: "withAlertsAdapter"
- args:
- - ref: "alertsAdapter"
- - name: "withOutputFieldName"
- args: ["message"]
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "alert"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.ww"
- - name: "withDocumentName"
- args:
- - "fireeye_alert"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "fireeye_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "geoEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "geoEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "hostEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "hostEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "joinBolt"
- className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt"
- configMethods:
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "${spout.kafka.topic.snort}"
+ - ref: "parser"
+ - ref: "writer"
streams:
- - name: "spout -> parser"
+ - name: "spout -> bolt"
from: "kafkaSpout"
to: "parserBolt"
grouping:
type: SHUFFLE
- - name: "parser -> host"
- from: "parserBolt"
- to: "hostEnrichmentBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "parser -> geo"
- from: "parserBolt"
- to: "geoEnrichmentBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "parser -> join"
- from: "parserBolt"
- to: "joinBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "geo -> join"
- from: "geoEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "host -> join"
- from: "hostEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "join -> alerts"
- from: "joinBolt"
- to: "alertsBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "alerts -> alertsIndexing"
- from: "alertsBolt"
- to: "alertsIndexingBolt"
- grouping:
- streamId: "message"
- type: SHUFFLE
- - name: "join -> indexing"
- from: "joinBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "alerts -> errors"
- from: "alertsBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/test.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/test.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/test.yaml
new file mode 100644
index 0000000..c014f86
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/test.yaml
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "fireeye-test"
+config:
+ topology.workers: 1
+
+components:
+ - id: "parser"
+ className: "org.apache.metron.parsing.parsers.BasicFireEyeParser"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
+ constructorArgs:
+ - "${kafka.broker}"
+ - id: "zkHosts"
+ className: "storm.kafka.ZkHosts"
+ constructorArgs:
+ - "${kafka.zk}"
+ - id: "kafkaConfig"
+ className: "storm.kafka.SpoutConfig"
+ constructorArgs:
+ # zookeeper hosts
+ - ref: "zkHosts"
+ # topic name
+ - "${spout.kafka.topic.snort}"
+ # zk root
+ - ""
+ # id
+ - "${spout.kafka.topic.snort}"
+ properties:
+ - name: "ignoreZkOffsets"
+ value: true
+ - name: "startOffsetTime"
+ value: -2
+
+spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - false
+ - id: "kafkaSpout"
+ className: "storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+
+bolts:
+ - id: "parserBolt"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "${spout.kafka.topic.snort}"
+ - ref: "parser"
+ - ref: "writer"
+
+streams:
+ - name: "spout -> bolt"
+ from: "kafkaSpout"
+ to: "parserBolt"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/local.yaml
deleted file mode 100644
index 2afba20..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/local.yaml
+++ /dev/null
@@ -1,192 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "ise-local"
-config:
- topology.workers: 1
-
-components:
- - id: "iseParser"
- className: "org.apache.metron.parsing.parsers.BasicIseParser"
- - id: "genericMessageFilter"
- className: "org.apache.metron.filters.GenericMessageFilter"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
-
-spouts:
- - id: "testingSpout"
- className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
- parallelism: 1
- configMethods:
- - name: "withFilename"
- args:
- - "SampleInput/ISESampleOutput"
- - name: "withRepeating"
- args:
- - true
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "iseParser"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "ise_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "ise_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "ise_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
-
-streams:
- - name: "spout -> parser"
- from: "testingSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
- - name: "parser -> indexing"
- from: "parserBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/remote.yaml
index 0196ae6..78cd779 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/remote.yaml
@@ -19,71 +19,12 @@ config:
topology.workers: 1
components:
- - id: "iseParser"
+ - id: "parser"
className: "org.apache.metron.parsing.parsers.BasicIseParser"
- - id: "genericMessageFilter"
- className: "org.apache.metron.filters.GenericMessageFilter"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
+ constructorArgs:
+ - "${kafka.broker}"
- id: "zkHosts"
className: "storm.kafka.ZkHosts"
constructorArgs:
@@ -94,18 +35,28 @@ components:
# zookeeper hosts
- ref: "zkHosts"
# topic name
- - "${spout.kafka.topic.ise}"
+ - "${spout.kafka.topic.snort}"
# zk root
- ""
# id
- - "${spout.kafka.topic.ise}"
+ - "${spout.kafka.topic.snort}"
properties:
- - name: "forceFromStart"
+ - name: "ignoreZkOffsets"
value: true
- name: "startOffsetTime"
value: -1
spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - true
- id: "kafkaSpout"
className: "storm.kafka.KafkaSpout"
constructorArgs:
@@ -113,94 +64,16 @@ spouts:
bolts:
- id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "iseParser"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "ise_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "ise_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "ise_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "${spout.kafka.topic.snort}"
+ - ref: "parser"
+ - ref: "writer"
streams:
- - name: "spout -> parser"
+ - name: "spout -> bolt"
from: "kafkaSpout"
to: "parserBolt"
grouping:
type: SHUFFLE
- - name: "parser -> indexing"
- from: "parserBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/test.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/test.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/test.yaml
new file mode 100644
index 0000000..4d6239c
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/test.yaml
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "ise-test"
+config:
+ topology.workers: 1
+
+components:
+ - id: "parser"
+ className: "org.apache.metron.parsing.parsers.BasicIseParser"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
+ constructorArgs:
+ - "${kafka.broker}"
+ - id: "zkHosts"
+ className: "storm.kafka.ZkHosts"
+ constructorArgs:
+ - "${kafka.zk}"
+ - id: "kafkaConfig"
+ className: "storm.kafka.SpoutConfig"
+ constructorArgs:
+ # zookeeper hosts
+ - ref: "zkHosts"
+ # topic name
+ - "${spout.kafka.topic.snort}"
+ # zk root
+ - ""
+ # id
+ - "${spout.kafka.topic.snort}"
+ properties:
+ - name: "ignoreZkOffsets"
+ value: true
+ - name: "startOffsetTime"
+ value: -2
+
+spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - false
+ - id: "kafkaSpout"
+ className: "storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+
+bolts:
+ - id: "parserBolt"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "${spout.kafka.topic.snort}"
+ - ref: "parser"
+ - ref: "writer"
+
+streams:
+ - name: "spout -> bolt"
+ from: "kafkaSpout"
+ to: "parserBolt"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/local.yaml
deleted file mode 100644
index 57a7344..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/local.yaml
+++ /dev/null
@@ -1,401 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "lancope-local"
-config:
- topology.workers: 1
-
-components:
- - id: "parser"
- className: "org.apache.metron.parsing.parsers.BasicLancopeParser"
- - id: "jdbcConfig"
- className: "org.apache.metron.enrichment.adapters.jdbc.MySqlConfig"
- properties:
- - name: "host"
- value: "${mysql.ip}"
- - name: "port"
- value: ${mysql.port}
- - name: "username"
- value: "${mysql.username}"
- - name: "password"
- value: "${mysql.password}"
- - name: "table"
- value: "GEO"
- - id: "geoEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
- configMethods:
- - name: "withJdbcConfig"
- args:
- - ref: "jdbcConfig"
- - id: "geoEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "geo"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "geoEnrichmentAdapter"
- - id: "hostEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
- constructorArgs:
- - '${org.apache.metron.enrichment.host.known_hosts}'
- - id: "hostEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "host"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "hostEnrichmentAdapter"
- - id: "enrichments"
- className: "java.util.ArrayList"
- configMethods:
- - name: "add"
- args:
- - ref: "geoEnrichment"
- - name: "add"
- args:
- - ref: "hostEnrichment"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "alertsConfig"
- className: "java.util.HashMap"
- configMethods:
- - name: "put"
- args: ["whitelist_table_name", "ip_whitelist"]
- - name: "put"
- args: ["blacklist_table_name", "ip_blacklist"]
- - name: "put"
- args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"]
- - name: "put"
- args: ["port", "2181"]
- - name: "put"
- args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"]
- - name: "put"
- args: ["_MAX_TIME_RETAIN_MINUTES", "1000"]
- - id: "alertsAdapter"
- className: "org.apache.metron.alerts.adapters.CIFAlertsAdapter"
- constructorArgs:
- - ref: "alertsConfig"
- - id: "alertsIdentifier"
- className: "org.json.simple.JSONObject"
- configMethods:
- - name: "put"
- args: ["environment", "local"]
- - name: "put"
- args: ["topology", "lancope"]
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
-
-spouts:
- - id: "testingSpout"
- className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
- parallelism: 1
- configMethods:
- - name: "withFilename"
- args:
- - "SampleInput/LancopeExampleOutput"
- - name: "withRepeating"
- args:
- - true
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "parser"
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "lancope_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "lancope_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsBolt"
- className: "org.apache.metron.alerts.TelemetryAlertsBolt"
- configMethods:
- - name: "withIdentifier"
- args:
- - ref: "alertsIdentifier"
- - name: "withMaxCacheSize"
- args: [1000]
- - name: "withMaxTimeRetain"
- args: [3600]
- - name: "withAlertsAdapter"
- args:
- - ref: "alertsAdapter"
- - name: "withOutputFieldName"
- args: ["message"]
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "alert"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.ww"
- - name: "withDocumentName"
- args:
- - "lancope_alert"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "lancope_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "geoEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "geoEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "hostEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "hostEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "joinBolt"
- className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt"
- configMethods:
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
-
-streams:
- - name: "spout -> parser"
- from: "testingSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
- - name: "parser -> host"
- from: "parserBolt"
- to: "hostEnrichmentBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "parser -> geo"
- from: "parserBolt"
- to: "geoEnrichmentBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "parser -> join"
- from: "parserBolt"
- to: "joinBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "geo -> join"
- from: "geoEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "host -> join"
- from: "hostEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "join -> alerts"
- from: "joinBolt"
- to: "alertsBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "alerts -> alertsIndexing"
- from: "alertsBolt"
- to: "alertsIndexingBolt"
- grouping:
- streamId: "message"
- type: SHUFFLE
- - name: "join -> indexing"
- from: "joinBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "alerts -> errors"
- from: "alertsBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
[8/9] incubator-metron git commit: METRON-56 Create unified
enrichment topology (merrimanr via cestella) closes
apache/incubator-metron#33
Posted by ce...@apache.org.
METRON-56 Create unified enrichment topology (merrimanr via cestella) closes apache/incubator-metron#33
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/9f96399d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/9f96399d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/9f96399d
Branch: refs/heads/master
Commit: 9f96399d9ecb252da13edf7bc44a366740945e85
Parents: 0e1055a
Author: merrimanr <me...@gmail.com>
Authored: Tue Mar 1 13:20:13 2016 -0500
Committer: cstella <ce...@gmail.com>
Committed: Tue Mar 1 13:20:13 2016 -0500
----------------------------------------------------------------------
.../metron/alerts/TelemetryAlertsBolt.java | 4 +-
metron-streaming/Metron-Common/pom.xml | 15 +
.../main/java/org/apache/metron/Constants.java | 27 +
.../metron/bolt/BulkMessageWriterBolt.java | 102 +
.../org/apache/metron/bolt/ConfiguredBolt.java | 91 +
.../java/org/apache/metron/bolt/JoinBolt.java | 39 +-
.../java/org/apache/metron/bolt/SplitBolt.java | 46 +-
.../org/apache/metron/domain/Enrichment.java | 21 +-
.../org/apache/metron/domain/SourceConfig.java | 88 +
.../metron/enrichment/EnrichmentConstants.java | 28 +
.../enrichment/EnrichmentSplitterBolt.java | 129 -
.../java/org/apache/metron/hbase/HBaseBolt.java | 5 +-
.../org/apache/metron/hbase/HTableProvider.java | 3 -
.../metron/helpers/topology/ErrorGenerator.java | 54 -
.../metron/helpers/topology/ErrorUtils.java | 64 +
.../metron/spout/pcap/HDFSWriterCallback.java | 169 ++
.../metron/spout/pcap/HDFSWriterConfig.java | 97 +
.../apache/metron/topology/TopologyUtils.java | 28 +
.../org/apache/metron/utils/ConfigUtils.java | 48 +
.../org/apache/metron/writer/HBaseWriter.java | 88 +
.../org/apache/metron/writer/PcapWriter.java | 52 +
.../writer/interfaces/BulkMessageWriter.java | 30 +
.../metron/writer/interfaces/MessageWriter.java | 27 +
.../src/main/java/storm/kafka/Callback.java | 26 +
.../java/storm/kafka/CallbackCollector.java | 182 ++
.../java/storm/kafka/CallbackKafkaSpout.java | 93 +
.../src/main/java/storm/kafka/EmitContext.java | 146 +
.../resources/config/source/bro-config.json | 13 +
.../resources/config/source/pcap-config.json | 13 +
.../resources/config/source/snort-config.json | 13 +
.../resources/config/source/yaf-config.json | 13 +
.../adapters/host/HostFromJSONListAdapter.java | 7 +-
.../enrichment/bolt/EnrichmentJoinBolt.java | 43 +-
.../enrichment/bolt/EnrichmentSplitterBolt.java | 140 +
.../enrichment/bolt/GenericEnrichmentBolt.java | 67 +-
.../enrichment/bolt/ThreatIntelJoinBolt.java | 41 +
.../bolt/ThreatIntelSplitterBolt.java | 40 +
.../enrichment/utils/EnrichmentUtils.java | 32 +
.../enrichment/utils/ThreatIntelUtils.java | 32 +
.../metron/indexing/AbstractIndexingBolt.java | 7 +-
.../metron/indexing/TelemetryIndexingBolt.java | 23 +-
.../adapters/ESTimedRotatingAdapter.java | 3 +-
.../metron/writer/ElasticSearchWriter.java | 95 +
.../org/apache/metron/writer/HdfsWriter.java | 44 +
metron-streaming/Metron-MessageParsers/pom.xml | 13 +
.../java/org/apache/metron/bolt/ParserBolt.java | 88 +
.../org/apache/metron/bolt/PcapParserBolt.java | 10 +-
.../apache/metron/bolt/TelemetryParserBolt.java | 12 +-
.../org/apache/metron/parser/MessageParser.java | 25 +
.../metron/parsing/parsers/GrokParser.java | 146 +
.../metron/parsing/parsers/PcapParser.java | 23 +-
.../org/apache/metron/writer/KafkaWriter.java | 79 +
.../src/main/resources/patterns/common | 96 +
.../src/main/resources/patterns/yaf | 113 +-
metron-streaming/Metron-Testing/pom.xml | 28 +-
.../util/integration/ComponentRunner.java | 15 +-
.../components/ElasticSearchComponent.java | 8 +-
.../components/KafkaWithZKComponent.java | 228 ++
.../util/integration/util/KafkaUtil.java | 41 +
.../org/apache/metron/utils/KafkaLoader.java | 88 +
.../apache/metron/utils/SourceConfigUtils.java | 95 +
.../Metron_Configs/topologies/asa/local.yaml | 401 ---
.../Metron_Configs/topologies/asa/remote.yaml | 385 +--
.../Metron_Configs/topologies/asa/test.yaml | 82 +
.../Metron_Configs/topologies/bro/local.yaml | 192 --
.../Metron_Configs/topologies/bro/remote.yaml | 176 +-
.../Metron_Configs/topologies/bro/test.yaml | 82 +
.../topologies/enrichment/remote.yaml | 331 +++
.../topologies/enrichment/test.yaml | 314 ++
.../topologies/fireeye/local.yaml | 401 ---
.../topologies/fireeye/remote.yaml | 382 +--
.../Metron_Configs/topologies/fireeye/test.yaml | 79 +
.../Metron_Configs/topologies/ise/local.yaml | 192 --
.../Metron_Configs/topologies/ise/remote.yaml | 177 +-
.../Metron_Configs/topologies/ise/test.yaml | 79 +
.../topologies/lancope/local.yaml | 401 ---
.../topologies/lancope/remote.yaml | 382 +--
.../Metron_Configs/topologies/lancope/test.yaml | 79 +
.../topologies/paloalto/local.yaml | 172 --
.../topologies/paloalto/remote.yaml | 155 +-
.../topologies/paloalto/test.yaml | 79 +
.../Metron_Configs/topologies/pcap/local.yaml | 22 +-
.../Metron_Configs/topologies/pcap/parse.yaml | 70 +
.../Metron_Configs/topologies/pcap/remote.yaml | 2 +-
.../Metron_Configs/topologies/snort/local.yaml | 195 --
.../Metron_Configs/topologies/snort/remote.yaml | 175 +-
.../Metron_Configs/topologies/snort/test.yaml | 79 +
.../topologies/sourcefire/local.yaml | 401 ---
.../topologies/sourcefire/remote.yaml | 382 +--
.../topologies/sourcefire/test.yaml | 79 +
.../Metron_Configs/topologies/yaf/local.yaml | 192 --
.../Metron_Configs/topologies/yaf/remote.yaml | 185 +-
.../Metron_Configs/topologies/yaf/test.yaml | 95 +
.../src/main/resources/SampleIndexed/YafIndexed | 10 +
.../src/main/resources/SampleInput/SnortOutput | 4 +-
.../main/resources/SampleInput/YafExampleOutput | 2695 +-----------------
.../src/main/resources/SampleParsed/SnortParsed | 3 +
.../resources/SampleParsed/YafExampleParsed | 10 +
.../integration/EnrichmentIntegrationTest.java | 195 ++
.../integration/ParserIntegrationTest.java | 155 +
.../integration/SnortIntegrationTest.java | 62 +
.../metron/integration/YafIntegrationTest.java | 62 +
.../integration/pcap/PcapIntegrationTest.java | 279 --
.../metron/integration/util/TestUtils.java | 38 +
pom.xml | 2 +-
105 files changed, 5213 insertions(+), 8156 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/TelemetryAlertsBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/TelemetryAlertsBolt.java b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/TelemetryAlertsBolt.java
index fd898e3..663ae40 100644
--- a/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/TelemetryAlertsBolt.java
+++ b/metron-streaming/Metron-Alerts/src/main/java/org/apache/metron/alerts/TelemetryAlertsBolt.java
@@ -34,7 +34,7 @@ import backtype.storm.tuple.Values;
import com.google.common.cache.CacheBuilder;
import org.apache.metron.alerts.interfaces.AlertsAdapter;
-import org.apache.metron.helpers.topology.ErrorGenerator;
+import org.apache.metron.helpers.topology.ErrorUtils;
import org.apache.metron.json.serialization.JSONEncoderHelper;
import org.apache.metron.metrics.MetricReporter;
@@ -245,7 +245,7 @@ public class TelemetryAlertsBolt extends AbstractAlertBolt {
*/
- JSONObject error = ErrorGenerator.generateErrorMessage(
+ JSONObject error = ErrorUtils.generateErrorMessage(
"Alerts problem: " + original_message, e);
_collector.emit("error", new Values(error));
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/pom.xml b/metron-streaming/Metron-Common/pom.xml
index 57a58d7..c4fc5aa 100644
--- a/metron-streaming/Metron-Common/pom.xml
+++ b/metron-streaming/Metron-Common/pom.xml
@@ -161,6 +161,21 @@
<artifactId>json-schema-validator</artifactId>
<version>${global_json_schema_validator_version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>2.7.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>flux-core</artifactId>
+ <version>${global_flux_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-kafka</artifactId>
+ <version>${global_storm_version}</version>
+ </dependency>
</dependencies>
<reporting>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/Constants.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/Constants.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/Constants.java
new file mode 100644
index 0000000..c6eafe9
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/Constants.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron;
+
+public class Constants {
+
+ public static final String ZOOKEEPER_ROOT = "/metron";
+ public static final String ZOOKEEPER_TOPOLOGY_ROOT = ZOOKEEPER_ROOT + "/topology";
+ public static final String SOURCE_TYPE = "source.type";
+ public static final String ENRICHMENT_TOPIC = "enrichments";
+ public static final String ERROR_STREAM = "error";
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
new file mode 100644
index 0000000..6d094ee
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.bolt;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Values;
+import org.apache.metron.Constants;
+import org.apache.metron.domain.SourceConfig;
+import org.apache.metron.helpers.topology.ErrorUtils;
+import org.apache.metron.topology.TopologyUtils;
+import org.apache.metron.writer.interfaces.BulkMessageWriter;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+public class BulkMessageWriterBolt extends ConfiguredBolt {
+
+ int count = 0;
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(BulkMessageWriterBolt.class);
+ private OutputCollector collector;
+ private BulkMessageWriter<JSONObject> bulkMessageWriter;
+ private Map<String, List<Tuple>> sourceTupleMap = new HashMap<>();
+ private Map<String, List<JSONObject>> sourceMessageMap = new HashMap<>();
+
+
+ public BulkMessageWriterBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
+
+ public BulkMessageWriterBolt withBulkMessageWriter(BulkMessageWriter<JSONObject> bulkMessageWriter) {
+ this.bulkMessageWriter = bulkMessageWriter;
+ return this;
+ }
+
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ this.collector = collector;
+ super.prepare(stormConf, context, collector);
+ bulkMessageWriter.init();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void execute(Tuple tuple) {
+ JSONObject message = (JSONObject) tuple.getValueByField("message");
+ String sourceType = TopologyUtils.getSourceType(message);
+ SourceConfig configuration = configurations.get(sourceType);
+ int batchSize = configuration != null ? configuration.getBatchSize() : 1;
+ List<Tuple> tupleList = sourceTupleMap.get(sourceType);
+ if (tupleList == null) tupleList = new ArrayList<>();
+ tupleList.add(tuple);
+ List<JSONObject> messageList = sourceMessageMap.get(sourceType);
+ if (messageList == null) messageList = new ArrayList<>();
+ messageList.add(message);
+ if (messageList.size() < batchSize) {
+ sourceTupleMap.put(sourceType, tupleList);
+ sourceMessageMap.put(sourceType, messageList);
+ } else {
+ try {
+ bulkMessageWriter.write(sourceType, configuration, tupleList, messageList);
+ for(Tuple t: tupleList) {
+ collector.ack(t);
+ }
+ } catch (Exception e) {
+ for(Tuple t: tupleList) {
+ collector.fail(t);
+ }
+ ErrorUtils.handleError(collector, e, Constants.ERROR_STREAM);
+ }
+ sourceTupleMap.remove(sourceType);
+ sourceMessageMap.remove(sourceType);
+ }
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declareStream("error", new Fields("message"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/ConfiguredBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/ConfiguredBolt.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/ConfiguredBolt.java
new file mode 100644
index 0000000..30c8e23
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/ConfiguredBolt.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.bolt;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.base.BaseRichBolt;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.recipes.cache.PathChildrenCache;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.log4j.Logger;
+import org.apache.metron.Constants;
+import org.apache.metron.domain.SourceConfig;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public abstract class ConfiguredBolt extends BaseRichBolt {
+
+ private static final Logger LOG = Logger.getLogger(ConfiguredBolt.class);
+
+ private String zookeeperUrl;
+
+ protected Map<String, SourceConfig> configurations = Collections.synchronizedMap(new HashMap<String, SourceConfig>());
+ private CuratorFramework client;
+ private PathChildrenCache cache;
+
+ public ConfiguredBolt(String zookeeperUrl) {
+ this.zookeeperUrl = zookeeperUrl;
+ }
+
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ client = CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy);
+ client.start();
+ cache = new PathChildrenCache(client, Constants.ZOOKEEPER_TOPOLOGY_ROOT, true);
+ PathChildrenCacheListener listener = new PathChildrenCacheListener() {
+ @Override
+ public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
+ if (event.getType().equals(PathChildrenCacheEvent.Type.CHILD_ADDED) || event.getType().equals(PathChildrenCacheEvent.Type.CHILD_UPDATED)) {
+ byte[] data = event.getData().getData();
+ if (data != null) {
+ SourceConfig temp = SourceConfig.load(data);
+ if (temp != null) {
+ String[] path = event.getData().getPath().split("/");
+ configurations.put(path[path.length - 1], temp);
+ }
+ }
+ }
+ }
+ };
+ cache.getListenable().addListener(listener);
+ try {
+ cache.start();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ try {
+ cache.close();
+ client.close();
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java
index dc84473..dac1c0a 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java
@@ -20,14 +20,12 @@ package org.apache.metron.bolt;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,18 +35,21 @@ import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
-public abstract class JoinBolt<V> extends BaseRichBolt {
+public abstract class JoinBolt<V> extends ConfiguredBolt {
private static final Logger LOG = LoggerFactory
.getLogger(JoinBolt.class);
protected OutputCollector collector;
- protected ImmutableSet<String> streamIds;
protected transient CacheLoader<String, Map<String, V>> loader;
protected transient LoadingCache<String, Map<String, V>> cache;
protected Long maxCacheSize;
protected Long maxTimeRetain;
+ public JoinBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
+
public JoinBolt withMaxCacheSize(long maxCacheSize) {
this.maxCacheSize = maxCacheSize;
return this;
@@ -61,6 +62,7 @@ public abstract class JoinBolt<V> extends BaseRichBolt {
@Override
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
+ super.prepare(map, topologyContext, outputCollector);
this.collector = outputCollector;
if (this.maxCacheSize == null)
throw new IllegalStateException("maxCacheSize must be specified");
@@ -74,9 +76,6 @@ public abstract class JoinBolt<V> extends BaseRichBolt {
cache = CacheBuilder.newBuilder().maximumSize(maxCacheSize)
.expireAfterWrite(maxTimeRetain, TimeUnit.MINUTES)
.build(loader);
- Set<String> temp = getStreamIds();
- temp.add("message");
- streamIds = ImmutableSet.copyOf(temp);
prepare(map, topologyContext);
}
@@ -85,26 +84,28 @@ public abstract class JoinBolt<V> extends BaseRichBolt {
public void execute(Tuple tuple) {
String streamId = tuple.getSourceStreamId();
String key = (String) tuple.getValueByField("key");
- V value = (V) tuple.getValueByField("message");
+ V message = (V) tuple.getValueByField("message");
try {
- Map<String, V> streamValueMap = cache.get(key);
- if (streamValueMap.containsKey(streamId)) {
+ Map<String, V> streamMessageMap = cache.get(key);
+ if (streamMessageMap.containsKey(streamId)) {
LOG.warn(String.format("Received key %s twice for " +
"stream %s", key, streamId));
}
- streamValueMap.put(streamId, value);
- Set<String> streamValueKeys = streamValueMap.keySet();
- if (streamValueKeys.size() == streamIds.size() && Sets.symmetricDifference
- (streamValueKeys, streamIds)
+ streamMessageMap.put(streamId, message);
+ Set<String> streamIds = getStreamIds(message);
+ Set<String> streamMessageKeys = streamMessageMap.keySet();
+ if (streamMessageKeys.size() == streamIds.size() && Sets.symmetricDifference
+ (streamMessageKeys, streamIds)
.isEmpty()) {
- collector.emit("message", tuple, new Values(key, joinValues
- (streamValueMap)));
+ collector.emit("message", tuple, new Values(key, joinMessages
+ (streamMessageMap)));
collector.ack(tuple);
cache.invalidate(key);
} else {
- cache.put(key, streamValueMap);
+ cache.put(key, streamMessageMap);
}
} catch (ExecutionException e) {
+ collector.reportError(e);
LOG.error(e.getMessage(), e);
}
}
@@ -116,7 +117,7 @@ public abstract class JoinBolt<V> extends BaseRichBolt {
public abstract void prepare(Map map, TopologyContext topologyContext);
- public abstract Set<String> getStreamIds();
+ public abstract Set<String> getStreamIds(V value);
- public abstract V joinValues(Map<String, V> streamValueMap);
+ public abstract V joinMessages(Map<String, V> streamMessageMap);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/SplitBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/SplitBolt.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/SplitBolt.java
index d3d2cf3..89e13a4 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/SplitBolt.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/SplitBolt.java
@@ -20,34 +20,33 @@ package org.apache.metron.bolt;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
-import com.google.common.collect.ImmutableSet;
-import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.UUID;
public abstract class SplitBolt<T> extends
- BaseRichBolt {
+ ConfiguredBolt {
protected OutputCollector collector;
- private Set<String> streamIds;
+
+ public SplitBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
@Override
public final void prepare(Map map, TopologyContext topologyContext,
OutputCollector outputCollector) {
+ super.prepare(map, topologyContext, outputCollector);
collector = outputCollector;
- streamIds = ImmutableSet.copyOf(getStreamIds());
prepare(map, topologyContext);
}
@Override
public final void execute(Tuple tuple) {
- emit(tuple, generateMessages(tuple));
+ emit(tuple, generateMessage(tuple));
}
@Override
@@ -60,24 +59,23 @@ public abstract class SplitBolt<T> extends
declareOther(declarer);
}
- public void emit(Tuple tuple, List<T> messages) {
- for(T message: messages) {
- String key = getKey(tuple, message);
- collector.emit("message", tuple, new Values(key, message));
- Map<String, T> streamValueMap = splitMessage(message);
- for (String streamId : streamIds) {
- T streamValue = streamValueMap.get(streamId);
- if (streamValue == null) {
- streamValue = getDefaultValue(streamId);
- }
- collector.emit(streamId, new Values(key, streamValue));
+ public void emit(Tuple tuple, T message) {
+ if (message == null) return;
+ String key = getKey(tuple, message);
+ collector.emit("message", tuple, new Values(key, message));
+ Map<String, T> streamMessageMap = splitMessage(message);
+ for (String streamId : streamMessageMap.keySet()) {
+ T streamMessage = streamMessageMap.get(streamId);
+ if (streamMessage == null) {
+ streamMessage = getDefaultMessage(streamId);
}
- collector.ack(tuple);
+ collector.emit(streamId, new Values(key, streamMessage));
}
- emitOther(tuple, messages);
+ collector.ack(tuple);
+ emitOther(tuple, message);
}
- protected T getDefaultValue(String streamId) {
+ protected T getDefaultMessage(String streamId) {
throw new IllegalArgumentException("Could not find a message for" +
" stream: " + streamId);
}
@@ -88,13 +86,13 @@ public abstract class SplitBolt<T> extends
public abstract String getKey(Tuple tuple, T message);
- public abstract List<T> generateMessages(Tuple tuple);
+ public abstract T generateMessage(Tuple tuple);
public abstract Map<String, T> splitMessage(T message);
public abstract void declareOther(OutputFieldsDeclarer declarer);
- public abstract void emitOther(Tuple tuple, List<T> messages);
+ public abstract void emitOther(Tuple tuple, T message);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java
index d75e9a3..7079d5c 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java
@@ -20,28 +20,25 @@ package org.apache.metron.domain;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import java.io.Serializable;
-import java.util.List;
public class Enrichment<T extends EnrichmentAdapter> implements Serializable {
- private String name;
- private List<String> fields;
+ private String type;
private T adapter;
- public String getName() {
- return name;
- }
+ public Enrichment() {}
- public void setName(String name) {
- this.name = name;
+ public Enrichment(String type, T adapter) {
+ this.type = type;
+ this.adapter = adapter;
}
- public List<String> getFields() {
- return fields;
+ public String getType() {
+ return type;
}
- public void setFields(List<String> fields) {
- this.fields = fields;
+ public void setType(String type) {
+ this.type = type;
}
public T getAdapter() {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/SourceConfig.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/SourceConfig.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/SourceConfig.java
new file mode 100644
index 0000000..8e1a960
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/SourceConfig.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.domain;
+
+import org.codehaus.jackson.map.ObjectMapper;
+import org.yaml.snakeyaml.TypeDescription;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.Constructor;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+
+public class SourceConfig {
+
+ final static ObjectMapper _mapper = new ObjectMapper();
+
+ private String index;
+ private Map<String, List<String>> enrichmentFieldMap;
+ private Map<String, List<String>> threatIntelFieldMap;
+ private int batchSize;
+
+ public String getIndex() {
+ return index;
+ }
+
+ public void setIndex(String index) {
+ this.index = index;
+ }
+
+ public Map<String, List<String>> getEnrichmentFieldMap() {
+ return enrichmentFieldMap;
+ }
+
+ public void setEnrichmentFieldMap(Map<String, List<String>> enrichmentFieldMap) {
+ this.enrichmentFieldMap = enrichmentFieldMap;
+ }
+
+ public Map<String, List<String>> getThreatIntelFieldMap() {
+ return threatIntelFieldMap;
+ }
+
+ public void setThreatIntelFieldMap(Map<String, List<String>> threatIntelFieldMap) {
+ this.threatIntelFieldMap = threatIntelFieldMap;
+ }
+
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ public static synchronized SourceConfig load(InputStream is) throws IOException {
+ SourceConfig ret = _mapper.readValue(is, SourceConfig.class);
+ return ret;
+ }
+
+ public static synchronized SourceConfig load(byte[] data) throws IOException {
+ return load( new ByteArrayInputStream(data));
+ }
+
+ public static synchronized SourceConfig load(String s, Charset c) throws IOException {
+ return load( s.getBytes(c));
+ }
+ public static synchronized SourceConfig load(String s) throws IOException {
+ return load( s, Charset.defaultCharset());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/enrichment/EnrichmentConstants.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/enrichment/EnrichmentConstants.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/enrichment/EnrichmentConstants.java
new file mode 100644
index 0000000..4f7be3b
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/enrichment/EnrichmentConstants.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment;
+
+public class EnrichmentConstants {
+
+
+
+ public static final String INDEX_NAME = "index.name";
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/enrichment/EnrichmentSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/enrichment/EnrichmentSplitterBolt.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/enrichment/EnrichmentSplitterBolt.java
deleted file mode 100644
index 967970f..0000000
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/enrichment/EnrichmentSplitterBolt.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.enrichment;
-
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Tuple;
-import com.google.common.base.Splitter;
-import org.apache.metron.bolt.SplitBolt;
-import org.apache.metron.domain.Enrichment;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-
-/**
- * Created by cstella on 2/10/16.
- */
-public class EnrichmentSplitterBolt extends SplitBolt<JSONObject> {
- protected static final Logger LOG = LoggerFactory.getLogger(EnrichmentSplitterBolt.class);
- protected List<Enrichment> enrichments = new ArrayList<>();
- protected String messageFieldName = "message";
- /**
- * @param enrichments A class for sending tuples to enrichment bolt
- * @return Instance of this class
- */
- public EnrichmentSplitterBolt withEnrichments(List<Enrichment> enrichments) {
- this.enrichments = enrichments;
- return this;
- }
- public EnrichmentSplitterBolt withMessageFieldName(String messageFieldName) {
- this.messageFieldName = messageFieldName;
- return this;
- }
- @Override
- public void prepare(Map map, TopologyContext topologyContext) {
-
- }
- @Override
- public String getKey(Tuple tuple, JSONObject message) {
- String key = null;
- try {
- key = tuple.getStringByField("key");
- }
- catch(Throwable t) {
- //swallowing this just in case.
- }
- if(key != null) {
- return key;
- }
- else {
- return UUID.randomUUID().toString();
- }
- }
-
- @Override
- public List<JSONObject> generateMessages(Tuple tuple) {
- return Arrays.asList((JSONObject)tuple.getValueByField(messageFieldName));
- }
-
- @Override
- public Set<String> getStreamIds() {
- Set<String> streamIds = new HashSet<>();
- for(Enrichment enrichment: enrichments) {
- streamIds.add(enrichment.getName());
- }
- return streamIds;
- }
- @SuppressWarnings("unchecked")
- @Override
- public Map<String, JSONObject> splitMessage(JSONObject message) {
-
- Map<String, JSONObject> streamMessageMap = new HashMap<>();
- for (Enrichment enrichment : enrichments) {
- List<String> fields = enrichment.getFields();
- if (fields != null && fields.size() > 0) {
- JSONObject enrichmentObject = new JSONObject();
- for (String field : fields) {
- enrichmentObject.put(field, getField(message,field));
- }
- streamMessageMap.put(enrichment.getName(), enrichmentObject);
- }
- }
- /*if(message != null && enrichments.size() != 1) {
- throw new RuntimeException("JSON: " + message.toJSONString() + " => " + streamMessageMap);
- }*/
- return streamMessageMap;
- }
-
- public Object getField(JSONObject object, String path) {
- Map ret = object;
- for(String node: Splitter.on('/').split(path)) {
- Object o = ret.get(node);
- if(o instanceof Map) {
- ret = (Map) o;
- }
- else {
- return o;
- }
- }
- return ret;
- }
-
- @Override
- public void declareOther(OutputFieldsDeclarer declarer) {
-
- }
-
- @Override
- public void emitOther(Tuple tuple, List<JSONObject> messages) {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/HBaseBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/HBaseBolt.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/HBaseBolt.java
index 7aa02c5..6caa016 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/HBaseBolt.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/HBaseBolt.java
@@ -20,7 +20,6 @@ package org.apache.metron.hbase;
import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
import java.util.Map;
import com.google.common.base.Function;
@@ -40,7 +39,7 @@ import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
-import org.apache.metron.helpers.topology.ErrorGenerator;
+import org.apache.metron.helpers.topology.ErrorUtils;
/**
* A Storm bolt for putting data into HBase.
@@ -136,7 +135,7 @@ public class HBaseBolt implements IRichBolt {
this.connector.put(p);
} catch (IOException ex) {
- JSONObject error = ErrorGenerator.generateErrorMessage(
+ JSONObject error = ErrorUtils.generateErrorMessage(
"Alerts problem: " + input.toString(), ex);
collector.emit("error", new Values(error));
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/HTableProvider.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/HTableProvider.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/HTableProvider.java
index 9055837..e454f04 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/HTableProvider.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/HTableProvider.java
@@ -23,9 +23,6 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import java.io.IOException;
-/**
- * Created by cstella on 2/11/16.
- */
public class HTableProvider implements TableProvider {
@Override
public HTableInterface getTable(Configuration config, String tableName) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/helpers/topology/ErrorGenerator.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/helpers/topology/ErrorGenerator.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/helpers/topology/ErrorGenerator.java
deleted file mode 100644
index 8ec940a..0000000
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/helpers/topology/ErrorGenerator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.helpers.topology;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.json.simple.JSONObject;
-
-public class ErrorGenerator {
-
- @SuppressWarnings("unchecked")
- public static JSONObject generateErrorMessage(String message, Exception e)
- {
- JSONObject error_message = new JSONObject();
-
- /*
- * Save full stack trace in object.
- */
- String stackTrace = ExceptionUtils.getStackTrace(e);
-
- String exception = e.toString();
-
- error_message.put("time", System.currentTimeMillis());
- try {
- error_message.put("hostname", InetAddress.getLocalHost().getHostName());
- } catch (UnknownHostException ex) {
- // TODO Auto-generated catch block
- ex.printStackTrace();
- }
-
- error_message.put("message", message);
- error_message.put("exception", exception);
- error_message.put("stack", stackTrace);
-
- return error_message;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/helpers/topology/ErrorUtils.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/helpers/topology/ErrorUtils.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/helpers/topology/ErrorUtils.java
new file mode 100644
index 0000000..b02cbaf
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/helpers/topology/ErrorUtils.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.helpers.topology;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.tuple.Values;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.metron.Constants;
+import org.json.simple.JSONObject;
+
+public class ErrorUtils {
+
+ @SuppressWarnings("unchecked")
+ public static JSONObject generateErrorMessage(String message, Throwable t)
+ {
+ JSONObject error_message = new JSONObject();
+
+ /*
+ * Save full stack trace in object.
+ */
+ String stackTrace = ExceptionUtils.getStackTrace(t);
+
+ String exception = t.toString();
+
+ error_message.put("time", System.currentTimeMillis());
+ try {
+ error_message.put("hostname", InetAddress.getLocalHost().getHostName());
+ } catch (UnknownHostException ex) {
+ // TODO Auto-generated catch block
+ ex.printStackTrace();
+ }
+
+ error_message.put("message", message);
+ error_message.put(Constants.SOURCE_TYPE, "error");
+ error_message.put("exception", exception);
+ error_message.put("stack", stackTrace);
+
+ return error_message;
+ }
+
+ public static void handleError(OutputCollector collector, Throwable t, String errorStream) {
+ JSONObject error = ErrorUtils.generateErrorMessage(t.getMessage(), t);
+ collector.emit(errorStream, new Values(error));
+ collector.reportError(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java
new file mode 100644
index 0000000..2c430d3
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/spout/pcap/HDFSWriterCallback.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.spout.pcap;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.log4j.Logger;
+import storm.kafka.Callback;
+import storm.kafka.EmitContext;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+public class HDFSWriterCallback implements Callback {
+ static final long serialVersionUID = 0xDEADBEEFL;
+ private static final Logger LOG = Logger.getLogger(HDFSWriterCallback.class);
+ public static final byte[] PCAP_GLOBAL_HEADER = new byte[] {
+ (byte) 0xd4, (byte) 0xc3, (byte) 0xb2, (byte) 0xa1, 0x02, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00
+ ,0x00, 0x00, 0x00, 0x00, (byte) 0xff, (byte) 0xff, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00
+ };
+
+ private static final List<Object> RET_TUPLE = ImmutableList.of((Object)Byte.valueOf((byte) 0x00), Byte.valueOf((byte)0x00));
+ private FileSystem fs;
+ private SequenceFile.Writer writer;
+ private HDFSWriterConfig config;
+ private long batchStartTime;
+ private long numWritten;
+ private EmitContext context;
+
+ public HDFSWriterCallback() {
+ //this.config = config;
+ }
+
+ public HDFSWriterCallback withConfig(HDFSWriterConfig config) {
+ LOG.info("Configured: " + config);
+ this.config = config;
+ return this;
+ }
+
+ @Override
+ public List<Object> apply(List<Object> tuple, EmitContext context) {
+
+ LongWritable ts = (LongWritable) tuple.get(0);
+ BytesWritable rawPacket = (BytesWritable)tuple.get(1);
+ try {
+ turnoverIfNecessary(ts.get());
+ writer.append(ts, headerize(rawPacket.getBytes()));
+ writer.hflush();
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ //drop? not sure..
+ }
+ return RET_TUPLE;
+ }
+
+ private static BytesWritable headerize(byte[] packet) {
+ byte[] ret = new byte[packet.length + PCAP_GLOBAL_HEADER.length];
+ int offset = 0;
+ System.arraycopy(PCAP_GLOBAL_HEADER, 0, ret, offset, PCAP_GLOBAL_HEADER.length);
+ offset += PCAP_GLOBAL_HEADER.length;
+ System.arraycopy(packet, 0, ret, offset, packet.length);
+ return new BytesWritable(ret);
+ }
+
+
+ private synchronized void turnoverIfNecessary(long ts) throws IOException {
+ long duration = ts - batchStartTime;
+ if(batchStartTime == 0L || duration > config.getMaxTimeMS() || numWritten > config.getNumPackets()) {
+ //turnover
+ Path path = getPath(ts);
+ if(writer != null) {
+ writer.close();
+ }
+ writer = SequenceFile.createWriter(new Configuration()
+ , SequenceFile.Writer.file(path)
+ , SequenceFile.Writer.keyClass(LongWritable.class)
+ , SequenceFile.Writer.valueClass(BytesWritable.class)
+ );
+ //reset state
+ LOG.info("Turning over and writing to " + path);
+ batchStartTime = ts;
+ numWritten = 0;
+ }
+ }
+
+ private Path getPath(long ts) {
+ String fileName = Joiner.on("_").join("pcap"
+ , "" + ts
+ , context.get(EmitContext.Type.UUID)
+ );
+ return new Path(config.getOutputPath(), fileName);
+ }
+
+ @Override
+ public void initialize(EmitContext context) {
+ this.context = context;
+ try {
+ fs = FileSystem.get(new Configuration());
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to create filesystem", e);
+ }
+ }
+
+ /**
+ * Closes this resource, relinquishing any underlying resources.
+ * This method is invoked automatically on objects managed by the
+ * {@code try}-with-resources statement.
+ * <p/>
+ * <p>While this interface method is declared to throw {@code
+ * Exception}, implementers are <em>strongly</em> encouraged to
+ * declare concrete implementations of the {@code close} method to
+ * throw more specific exceptions, or to throw no exception at all
+ * if the close operation cannot fail.
+ * <p/>
+ * <p><em>Implementers of this interface are also strongly advised
+ * to not have the {@code close} method throw {@link
+ * InterruptedException}.</em>
+ * <p/>
+ * This exception interacts with a thread's interrupted status,
+ * and runtime misbehavior is likely to occur if an {@code
+ * InterruptedException} is {@linkplain Throwable#addSuppressed
+ * suppressed}.
+ * <p/>
+ * More generally, if it would cause problems for an
+ * exception to be suppressed, the {@code AutoCloseable.close}
+ * method should not throw it.
+ * <p/>
+ * <p>Note that unlike the {@link Closeable#close close}
+ * method of {@link Closeable}, this {@code close} method
+ * is <em>not</em> required to be idempotent. In other words,
+ * calling this {@code close} method more than once may have some
+ * visible side effect, unlike {@code Closeable.close} which is
+ * required to have no effect if called more than once.
+ * <p/>
+ * However, implementers of this interface are strongly encouraged
+ * to make their {@code close} methods idempotent.
+ *
+ * @throws Exception if this resource cannot be closed
+ */
+ @Override
+ public void close() throws Exception {
+ if(writer != null) {
+ writer.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/spout/pcap/HDFSWriterConfig.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/spout/pcap/HDFSWriterConfig.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/spout/pcap/HDFSWriterConfig.java
new file mode 100644
index 0000000..ccfc884
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/spout/pcap/HDFSWriterConfig.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.spout.pcap;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+public class HDFSWriterConfig implements Serializable {
+ static final long serialVersionUID = 0xDEADBEEFL;
+ private long numPackets;
+ private long maxTimeMS;
+ private String outputPath;
+ private String zookeeperQuorum;
+
+ public HDFSWriterConfig withOutputPath(String path) {
+ outputPath = path;
+ return this;
+ }
+
+ public HDFSWriterConfig withNumPackets(long n) {
+ numPackets = n;
+ return this;
+ }
+
+ public HDFSWriterConfig withMaxTimeMS(long t) {
+ maxTimeMS = t;
+ return this;
+ }
+
+ public HDFSWriterConfig withZookeeperQuorum(String zookeeperQuorum) {
+ this.zookeeperQuorum = zookeeperQuorum;
+ return this;
+ }
+
+ public List<String> getZookeeperServers() {
+ List<String> out = new ArrayList<>();
+ if(zookeeperQuorum != null) {
+ for (String hostPort : Splitter.on(',').split(zookeeperQuorum)) {
+ Iterable<String> tokens = Splitter.on(':').split(hostPort);
+ String host = Iterables.getFirst(tokens, null);
+ if(host != null) {
+ out.add(host);
+ }
+ }
+ }
+ return out;
+ }
+
+ public Integer getZookeeperPort() {
+ if(zookeeperQuorum != null) {
+ String hostPort = Iterables.getFirst(Splitter.on(',').split(zookeeperQuorum), null);
+ String portStr = Iterables.getLast(Splitter.on(':').split(hostPort));
+ return Integer.parseInt(portStr);
+ }
+ return null;
+ }
+
+ public String getOutputPath() {
+ return outputPath;
+ }
+
+ public long getNumPackets() {
+ return numPackets;
+ }
+
+ public long getMaxTimeMS() {
+ return maxTimeMS;
+ }
+
+ @Override
+ public String toString() {
+ return "HDFSWriterConfig{" +
+ "numPackets=" + numPackets +
+ ", maxTimeMS=" + maxTimeMS +
+ ", outputPath='" + outputPath + '\'' +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/topology/TopologyUtils.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/topology/TopologyUtils.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/topology/TopologyUtils.java
new file mode 100644
index 0000000..581d74f
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/topology/TopologyUtils.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.topology;
+
+import org.apache.metron.Constants;
+import org.json.simple.JSONObject;
+
+public class TopologyUtils {
+
+ public static String getSourceType(JSONObject message) {
+ return (String) message.get(Constants.SOURCE_TYPE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/utils/ConfigUtils.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/utils/ConfigUtils.java
new file mode 100644
index 0000000..7f5afe9
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/utils/ConfigUtils.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.utils;
+
+import java.lang.reflect.InvocationTargetException;
+
+public class ConfigUtils<T> {
+
+ public static <T> T createInstance(String className, T defaultClass) {
+ T instance;
+ if(className == null || className.length() == 0 || className.charAt(0) == '$') {
+ return defaultClass;
+ }
+ else {
+ try {
+ Class<? extends T> clazz = (Class<? extends T>) Class.forName(className);
+ instance = clazz.getConstructor().newInstance();
+ } catch (InstantiationException e) {
+ throw new IllegalStateException("Unable to instantiate connector.", e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalStateException("Unable to instantiate connector: illegal access", e);
+ } catch (InvocationTargetException e) {
+ throw new IllegalStateException("Unable to instantiate connector", e);
+ } catch (NoSuchMethodException e) {
+ throw new IllegalStateException("Unable to instantiate connector: no such method", e);
+ } catch (ClassNotFoundException e) {
+ throw new IllegalStateException("Unable to instantiate connector: class not found", e);
+ }
+ }
+ return instance;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/HBaseWriter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/HBaseWriter.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/HBaseWriter.java
new file mode 100644
index 0000000..b257b24
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/HBaseWriter.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer;
+
+import backtype.storm.tuple.Tuple;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.domain.SourceConfig;
+import org.apache.metron.hbase.HTableProvider;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.utils.ConfigUtils;
+import org.apache.metron.writer.interfaces.MessageWriter;
+import org.json.simple.JSONObject;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Map;
+
+public abstract class HBaseWriter implements MessageWriter<JSONObject>, Serializable {
+
+ private String tableName;
+ private String connectorImpl;
+ private TableProvider provider;
+ private HTableInterface table;
+
+ public HBaseWriter(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public HBaseWriter withProviderImpl(String connectorImpl) {
+ this.connectorImpl = connectorImpl;
+ return this;
+ }
+
+ @Override
+ public void init() {
+ final Configuration config = HBaseConfiguration.create();
+ try {
+ provider = ConfigUtils.createInstance(connectorImpl, new HTableProvider());
+ table = provider.getTable(config, tableName);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void write(String sourceType, SourceConfig configuration, Tuple tuple, JSONObject message) throws Exception {
+ Put put = new Put(getKey(tuple, message));
+ Map<String, byte[]> values = getValues(tuple, message);
+ for(String column: values.keySet()) {
+ String[] columnParts = column.split(":");
+ long timestamp = getTimestamp(tuple, message);
+ if (timestamp > -1) {
+ put.addColumn(Bytes.toBytes(columnParts[0]), Bytes.toBytes(columnParts[1]), timestamp, values.get(column));
+ } else {
+ put.addColumn(Bytes.toBytes(columnParts[0]), Bytes.toBytes(columnParts[1]), values.get(column));
+ }
+ }
+ table.put(put);
+ }
+
+ @Override
+ public void close() throws Exception {
+ table.close();
+ }
+
+ public abstract byte[] getKey(Tuple tuple, JSONObject message);
+ public abstract long getTimestamp(Tuple tuple, JSONObject message);
+ public abstract Map<String, byte[]> getValues(Tuple tuple, JSONObject message);
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/PcapWriter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/PcapWriter.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/PcapWriter.java
new file mode 100644
index 0000000..b5ab587
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/PcapWriter.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer;
+
+import backtype.storm.tuple.Tuple;
+import org.json.simple.JSONObject;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class PcapWriter extends HBaseWriter {
+
+ private String column;
+
+ public PcapWriter(String tableName, String column) {
+ super(tableName);
+ this.column = column;
+ }
+
+ @Override
+ public byte[] getKey(Tuple tuple, JSONObject message) {
+ String key = (String) message.get("pcap_id");
+ return key.getBytes();
+ }
+
+ @Override
+ public long getTimestamp(Tuple tuple, JSONObject message) {
+ return (long) message.get("ts_micro");
+ }
+
+ @Override
+ public Map<String, byte[]> getValues(Tuple tuple, JSONObject message) {
+ Map<String, byte[]> values = new HashMap<>();
+ values.put(column, tuple.getBinary(0));
+ return values;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/interfaces/BulkMessageWriter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/interfaces/BulkMessageWriter.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/interfaces/BulkMessageWriter.java
new file mode 100644
index 0000000..90c0261
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/interfaces/BulkMessageWriter.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer.interfaces;
+
+import backtype.storm.tuple.Tuple;
+import org.apache.metron.domain.SourceConfig;
+
+import java.util.List;
+
+public interface BulkMessageWriter<T> extends AutoCloseable {
+
+ void init();
+ void write(String sourceType, SourceConfig configuration, List<Tuple> tuples, List<T> messages) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/interfaces/MessageWriter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/interfaces/MessageWriter.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/interfaces/MessageWriter.java
new file mode 100644
index 0000000..12de836
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/writer/interfaces/MessageWriter.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer.interfaces;
+
+import backtype.storm.tuple.Tuple;
+import org.apache.metron.domain.SourceConfig;
+
+public interface MessageWriter<T> extends AutoCloseable {
+
+ void init();
+ void write(String sourceType, SourceConfig configuration, Tuple tuple, T message) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/storm/kafka/Callback.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/storm/kafka/Callback.java b/metron-streaming/Metron-Common/src/main/java/storm/kafka/Callback.java
new file mode 100644
index 0000000..ff05c29
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/storm/kafka/Callback.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package storm.kafka;
+
+import java.io.Serializable;
+import java.util.List;
+
+public interface Callback extends AutoCloseable, Serializable {
+ List<Object> apply(List<Object> tuple, EmitContext context);
+ void initialize(EmitContext context);
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/storm/kafka/CallbackCollector.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/storm/kafka/CallbackCollector.java b/metron-streaming/Metron-Common/src/main/java/storm/kafka/CallbackCollector.java
new file mode 100644
index 0000000..485da5a
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/storm/kafka/CallbackCollector.java
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package storm.kafka;
+
+import backtype.storm.spout.ISpoutOutputCollector;
+import backtype.storm.spout.SpoutOutputCollector;
+
+import java.io.Serializable;
+import java.util.List;
+
+public class CallbackCollector extends SpoutOutputCollector implements Serializable {
+ static final long serialVersionUID = 0xDEADBEEFL;
+ Callback _callback;
+ SpoutOutputCollector _delegate;
+ EmitContext _context;
+ public CallbackCollector(Callback callback, SpoutOutputCollector collector, EmitContext context) {
+ super(collector);
+ this._callback = callback;
+ this._delegate = collector;
+ this._context = context;
+ }
+
+
+ /**
+ * Emits a new tuple to the specified output stream with the given message ID.
+ * When Storm detects that this tuple has been fully processed, or has failed
+ * to be fully processed, the spout will receive an ack or fail callback respectively
+ * with the messageId as long as the messageId was not null. If the messageId was null,
+ * Storm will not track the tuple and no callback will be received. The emitted values must be
+ * immutable.
+ *
+ * @param streamId
+ * @param tuple
+ * @param messageId
+ * @return the list of task ids that this tuple was sent to
+ */
+ @Override
+ public List<Integer> emit(String streamId, List<Object> tuple, Object messageId) {
+ List<Object> t = _callback.apply(tuple, _context.cloneContext().with(EmitContext.Type.MESSAGE_ID, messageId)
+ .with(EmitContext.Type.STREAM_ID, streamId)
+ );
+ return _delegate.emit(streamId, t, messageId);
+ }
+
+ /**
+ * Emits a new tuple to the default output stream with the given message ID.
+ * When Storm detects that this tuple has been fully processed, or has failed
+ * to be fully processed, the spout will receive an ack or fail callback respectively
+ * with the messageId as long as the messageId was not null. If the messageId was null,
+ * Storm will not track the tuple and no callback will be received. The emitted values must be
+ * immutable.
+ *
+ * @param tuple
+ * @param messageId
+ * @return the list of task ids that this tuple was sent to
+ */
+ @Override
+ public List<Integer> emit(List<Object> tuple, Object messageId) {
+ List<Object> t = _callback.apply(tuple, _context.cloneContext().with(EmitContext.Type.MESSAGE_ID, messageId));
+ return super.emit(t, messageId);
+ }
+
+ /**
+ * Emits a tuple to the default output stream with a null message id. Storm will
+ * not track this message so ack and fail will never be called for this tuple. The
+ * emitted values must be immutable.
+ *
+ * @param tuple
+ */
+ @Override
+ public List<Integer> emit(List<Object> tuple) {
+ List<Object> t = _callback.apply(tuple, _context.cloneContext());
+ return super.emit(t);
+ }
+
+ /**
+ * Emits a tuple to the specified output stream with a null message id. Storm will
+ * not track this message so ack and fail will never be called for this tuple. The
+ * emitted values must be immutable.
+ *
+ * @param streamId
+ * @param tuple
+ */
+ @Override
+ public List<Integer> emit(String streamId, List<Object> tuple) {
+ List<Object> t = _callback.apply(tuple, _context.cloneContext().with(EmitContext.Type.STREAM_ID, streamId));
+ return super.emit(streamId, t);
+ }
+
+ /**
+ * Emits a tuple to the specified task on the specified output stream. This output
+ * stream must have been declared as a direct stream, and the specified task must
+ * use a direct grouping on this stream to receive the message. The emitted values must be
+ * immutable.
+ *
+ * @param taskId
+ * @param streamId
+ * @param tuple
+ * @param messageId
+ */
+ @Override
+ public void emitDirect(int taskId, String streamId, List<Object> tuple, Object messageId) {
+ List<Object> t = _callback.apply(tuple, _context.cloneContext().with(EmitContext.Type.STREAM_ID, streamId)
+ .with(EmitContext.Type.MESSAGE_ID, messageId)
+ .with(EmitContext.Type.TASK_ID, new Integer(taskId))
+ );
+ super.emitDirect(taskId, streamId, t, messageId);
+ }
+
+ /**
+ * Emits a tuple to the specified task on the default output stream. This output
+ * stream must have been declared as a direct stream, and the specified task must
+ * use a direct grouping on this stream to receive the message. The emitted values must be
+ * immutable.
+ *
+ * @param taskId
+ * @param tuple
+ * @param messageId
+ */
+ @Override
+ public void emitDirect(int taskId, List<Object> tuple, Object messageId) {
+ List<Object> t = _callback.apply(tuple, _context.cloneContext().with(EmitContext.Type.MESSAGE_ID, messageId)
+ .with(EmitContext.Type.TASK_ID, new Integer(taskId))
+ );
+ super.emitDirect(taskId, t, messageId);
+ }
+
+ /**
+ * Emits a tuple to the specified task on the specified output stream. This output
+ * stream must have been declared as a direct stream, and the specified task must
+ * use a direct grouping on this stream to receive the message. The emitted values must be
+ * immutable.
+ * <p/>
+ * <p> Because no message id is specified, Storm will not track this message
+ * so ack and fail will never be called for this tuple.</p>
+ *
+ * @param taskId
+ * @param streamId
+ * @param tuple
+ */
+ @Override
+ public void emitDirect(int taskId, String streamId, List<Object> tuple) {
+ List<Object> t = _callback.apply(tuple, _context.cloneContext().with(EmitContext.Type.STREAM_ID, streamId)
+ .with(EmitContext.Type.TASK_ID, new Integer(taskId))
+ );
+ super.emitDirect(taskId, streamId, t);
+ }
+
+ /**
+ * Emits a tuple to the specified task on the default output stream. This output
+ * stream must have been declared as a direct stream, and the specified task must
+ * use a direct grouping on this stream to receive the message. The emitted values must be
+ * immutable.
+ * <p/>
+ * <p> Because no message id is specified, Storm will not track this message
+ * so ack and fail will never be called for this tuple.</p>
+ *
+ * @param taskId
+ * @param tuple
+ */
+ @Override
+ public void emitDirect(int taskId, List<Object> tuple) {
+
+ List<Object> t = _callback.apply(tuple, _context.cloneContext().with(EmitContext.Type.TASK_ID, new Integer(taskId)));
+ super.emitDirect(taskId, t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/storm/kafka/CallbackKafkaSpout.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/storm/kafka/CallbackKafkaSpout.java b/metron-streaming/Metron-Common/src/main/java/storm/kafka/CallbackKafkaSpout.java
new file mode 100644
index 0000000..431bdf9
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/storm/kafka/CallbackKafkaSpout.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package storm.kafka;
+
+import backtype.storm.Config;
+import backtype.storm.metric.api.IMetric;
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+import storm.kafka.*;
+
+import java.util.*;
+
+public class CallbackKafkaSpout extends KafkaSpout {
+ static final long serialVersionUID = 0xDEADBEEFL;
+ Class<? extends Callback> callbackClazz;
+ Callback _callback;
+ EmitContext _context;
+ public CallbackKafkaSpout(SpoutConfig spoutConfig, String callbackClass) {
+ this(spoutConfig, toCallbackClass(callbackClass));
+ }
+
+ public CallbackKafkaSpout(SpoutConfig spoutConf, Class<? extends Callback> callback) {
+ super(spoutConf);
+ callbackClazz = callback;
+ }
+
+ public void initialize() {
+ _callback = createCallback(callbackClazz);
+ _context = new EmitContext().with(EmitContext.Type.SPOUT_CONFIG, _spoutConfig)
+ .with(EmitContext.Type.UUID, _uuid);
+ _callback.initialize(_context);
+ }
+
+
+ private static Class<? extends Callback> toCallbackClass(String callbackClass) {
+ try{
+ return (Class<? extends Callback>) Callback.class.forName(callbackClass);
+ }
+ catch (ClassNotFoundException e) {
+ throw new RuntimeException(callbackClass + " not found", e);
+ }
+ }
+
+ protected Callback createCallback(Class<? extends Callback> callbackClass) {
+ try {
+ return callbackClass.newInstance();
+ } catch (InstantiationException e) {
+ throw new RuntimeException("Unable to instantiate callback", e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException("Illegal access", e);
+ }
+ }
+
+ @Override
+ public void open(Map conf, final TopologyContext context, final SpoutOutputCollector collector) {
+ if(_callback == null) {
+ initialize();
+ }
+ super.open( conf, context
+ , new CallbackCollector(_callback, collector
+ ,_context.cloneContext().with(EmitContext.Type.OPEN_CONFIG, conf)
+ .with(EmitContext.Type.TOPOLOGY_CONTEXT, context)
+ )
+ );
+ }
+
+ @Override
+ public void close() {
+ super.close();
+ if(_callback != null) {
+ try {
+ _callback.close();
+ } catch (Exception e) {
+ throw new IllegalStateException("Unable to close callback", e);
+ }
+ }
+ }
+}
[7/9] incubator-metron git commit: METRON-56 Create unified
enrichment topology (merrimanr via cestella) closes
apache/incubator-metron#33
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/main/java/storm/kafka/EmitContext.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/storm/kafka/EmitContext.java b/metron-streaming/Metron-Common/src/main/java/storm/kafka/EmitContext.java
new file mode 100644
index 0000000..1f9ef59
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/storm/kafka/EmitContext.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package storm.kafka;
+
+import backtype.storm.task.TopologyContext;
+
+import java.io.Serializable;
+import java.util.EnumMap;
+import java.util.Map;
+
+public class EmitContext implements Cloneable,Serializable {
+ static final long serialVersionUID = 0xDEADBEEFL;
+
+ public enum Type{
+ MESSAGE_ID(PartitionManager.KafkaMessageId.class)
+ ,STREAM_ID(String.class)
+ ,TASK_ID(Integer.class)
+ ,UUID(String.class)
+ ,SPOUT_CONFIG(SpoutConfig.class)
+ ,OPEN_CONFIG(Map.class)
+ ,TOPOLOGY_CONTEXT(TopologyContext.class)
+ ;
+ Class<?> clazz;
+ Type(Class<?> clazz) {
+ this.clazz= clazz;
+ }
+
+ public Class<?> clazz() {
+ return clazz;
+ }
+ }
+ public EmitContext() {
+ this(new EnumMap<>(Type.class));
+ }
+ public EmitContext(EnumMap<Type, Object> context) {
+ _context = context;
+ }
+ private EnumMap<Type, Object> _context;
+
+ public <T> EmitContext with(Type t, T o ) {
+ _context.put(t, t.clazz().cast(o));
+ return this;
+ }
+ public <T> void add(Type t, T o ) {
+ with(t, o);
+ }
+
+ public <T> T get(Type t) {
+ Object o = _context.get(t);
+ if(o == null) {
+ return null;
+ }
+ else {
+ return (T) o;
+ }
+ }
+
+ public EmitContext cloneContext() {
+ try {
+ return (EmitContext)this.clone();
+ } catch (CloneNotSupportedException e) {
+ throw new RuntimeException("Unable to clone emit context.", e);
+ }
+ }
+
+ /**
+ * Creates and returns a copy of this object. The precise meaning
+ * of "copy" may depend on the class of the object. The general
+ * intent is that, for any object {@code x}, the expression:
+ * <blockquote>
+ * <pre>
+ * x.clone() != x</pre></blockquote>
+ * will be true, and that the expression:
+ * <blockquote>
+ * <pre>
+ * x.clone().getClass() == x.getClass()</pre></blockquote>
+ * will be {@code true}, but these are not absolute requirements.
+ * While it is typically the case that:
+ * <blockquote>
+ * <pre>
+ * x.clone().equals(x)</pre></blockquote>
+ * will be {@code true}, this is not an absolute requirement.
+ *
+ * By convention, the returned object should be obtained by calling
+ * {@code super.clone}. If a class and all of its superclasses (except
+ * {@code Object}) obey this convention, it will be the case that
+ * {@code x.clone().getClass() == x.getClass()}.
+ *
+ * By convention, the object returned by this method should be independent
+ * of this object (which is being cloned). To achieve this independence,
+ * it may be necessary to modify one or more fields of the object returned
+ * by {@code super.clone} before returning it. Typically, this means
+ * copying any mutable objects that comprise the internal "deep structure"
+ * of the object being cloned and replacing the references to these
+ * objects with references to the copies. If a class contains only
+ * primitive fields or references to immutable objects, then it is usually
+ * the case that no fields in the object returned by {@code super.clone}
+ * need to be modified.
+ *
+ * The method {@code clone} for class {@code Object} performs a
+ * specific cloning operation. First, if the class of this object does
+ * not implement the interface {@code Cloneable}, then a
+ * {@code CloneNotSupportedException} is thrown. Note that all arrays
+ * are considered to implement the interface {@code Cloneable} and that
+ * the return type of the {@code clone} method of an array type {@code T[]}
+ * is {@code T[]} where T is any reference or primitive type.
+ * Otherwise, this method creates a new instance of the class of this
+ * object and initializes all its fields with exactly the contents of
+ * the corresponding fields of this object, as if by assignment; the
+ * contents of the fields are not themselves cloned. Thus, this method
+ * performs a "shallow copy" of this object, not a "deep copy" operation.
+ *
+ * The class {@code Object} does not itself implement the interface
+ * {@code Cloneable}, so calling the {@code clone} method on an object
+ * whose class is {@code Object} will result in throwing an
+ * exception at run time.
+ *
+ * @return a clone of this instance.
+ * @throws CloneNotSupportedException if the object's class does not
+ * support the {@code Cloneable} interface. Subclasses
+ * that override the {@code clone} method can also
+ * throw this exception to indicate that an instance cannot
+ * be cloned.
+ * @see Cloneable
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ EmitContext context = new EmitContext(_context.clone());
+ return context;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/test/resources/config/source/bro-config.json
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/test/resources/config/source/bro-config.json b/metron-streaming/Metron-Common/src/test/resources/config/source/bro-config.json
new file mode 100644
index 0000000..fcbfc03
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/test/resources/config/source/bro-config.json
@@ -0,0 +1,13 @@
+{
+ "index": "bro",
+ "batchSize": 5,
+ "enrichmentFieldMap":
+ {
+ "geo": ["id.orig_h"],
+ "host": ["id.orig_h"]
+ },
+ "threatIntelFieldMap":
+ {
+ "ip": ["id.orig_h"]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/test/resources/config/source/pcap-config.json
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/test/resources/config/source/pcap-config.json b/metron-streaming/Metron-Common/src/test/resources/config/source/pcap-config.json
new file mode 100644
index 0000000..82c7c5e
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/test/resources/config/source/pcap-config.json
@@ -0,0 +1,13 @@
+{
+ "index": "pcap",
+ "batchSize": 5,
+ "enrichmentFieldMap":
+ {
+ "geo": ["ip_src_addr", "ip_dst_addr"],
+ "host": ["ip_src_addr", "ip_dst_addr"]
+ },
+ "threatIntelFieldMap":
+ {
+ "ip": ["ip_src_addr", "ip_dst_addr"]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/test/resources/config/source/snort-config.json
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/test/resources/config/source/snort-config.json b/metron-streaming/Metron-Common/src/test/resources/config/source/snort-config.json
new file mode 100644
index 0000000..ceb441e
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/test/resources/config/source/snort-config.json
@@ -0,0 +1,13 @@
+{
+ "index": "snort",
+ "batchSize": 1,
+ "enrichmentFieldMap":
+ {
+ "geo": ["src", "dst"],
+ "host": ["src", "dst"]
+ },
+ "threatIntelFieldMap":
+ {
+ "ip": ["src", "dst"]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Common/src/test/resources/config/source/yaf-config.json
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/test/resources/config/source/yaf-config.json b/metron-streaming/Metron-Common/src/test/resources/config/source/yaf-config.json
new file mode 100644
index 0000000..abf4ff4
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/test/resources/config/source/yaf-config.json
@@ -0,0 +1,13 @@
+{
+ "index": "yaf",
+ "batchSize": 5,
+ "enrichmentFieldMap":
+ {
+ "geo": ["sip", "dip"],
+ "host": ["sip", "dip"]
+ },
+ "threatIntelFieldMap":
+ {
+ "ip": ["sip", "dip"]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java
index f4d10c1..c55b918 100644
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapter.java
@@ -66,7 +66,12 @@ public class HostFromJSONListAdapter extends AbstractHostAdapter {
return new JSONObject();
JSONObject enrichment = new JSONObject();
- enrichment.put("known_info", _known_hosts.get(metadata));
+ String prefix = "known_info.";
+ JSONObject knownInfo = _known_hosts.get(metadata);
+ for(Object key: knownInfo.keySet()) {
+ enrichment.put(prefix + key, knownInfo.get(key));
+ }
+ //enrichment.put("known_info", _known_hosts.get(metadata));
return enrichment;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
index 866a009..10e1e71 100644
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
@@ -20,6 +20,7 @@ package org.apache.metron.enrichment.bolt;
import backtype.storm.task.TopologyContext;
import org.apache.metron.bolt.JoinBolt;
import org.apache.metron.domain.Enrichment;
+import org.apache.metron.topology.TopologyUtils;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,20 +35,14 @@ public class EnrichmentJoinBolt extends JoinBolt<JSONObject> {
protected static final Logger LOG = LoggerFactory
.getLogger(EnrichmentJoinBolt.class);
- protected List<Enrichment> enrichments;
+ private List<Enrichment> enrichments;
- protected String type = "enrichment";
- /**
- * @param enrichments A class for sending tuples to enrichment bolt
- * @return Instance of this class
- */
- public EnrichmentJoinBolt withEnrichments(List<Enrichment> enrichments) {
- this.enrichments = enrichments;
- return this;
+ public EnrichmentJoinBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
}
- public EnrichmentJoinBolt withType(String type) {
- this.type = type;
+ public EnrichmentJoinBolt withEnrichments(List<Enrichment> enrichments) {
+ this.enrichments = enrichments;
return this;
}
@@ -57,29 +52,27 @@ public class EnrichmentJoinBolt extends JoinBolt<JSONObject> {
}
@Override
- public Set<String> getStreamIds() {
+ public Set<String> getStreamIds(JSONObject message) {
Set<String> streamIds = new HashSet<>();
- for(Enrichment enrichment: enrichments) {
- streamIds.add(enrichment.getName());
+ String sourceType = TopologyUtils.getSourceType(message);
+ for (String enrichmentType : getFieldMap(sourceType).keySet()) {
+ streamIds.add(enrichmentType);
}
+ streamIds.add("message");
return streamIds;
}
@Override
- public JSONObject joinValues(Map<String, JSONObject> streamValueMap) {
+ public JSONObject joinMessages(Map<String, JSONObject> streamMessageMap) {
JSONObject message = new JSONObject();
- if(streamValueMap.get("message").containsKey("message")) {
- message = streamValueMap.get("message");
+ for (String key : streamMessageMap.keySet()) {
+ message.putAll(streamMessageMap.get(key));
}
- else {
- message.put("message", streamValueMap.get("message"));
- }
- JSONObject enrichment = new JSONObject();
- for(String streamId: getStreamIds()) {
- enrichment.put(streamId, streamValueMap.get(streamId));
- }
- message.put(type, enrichment);
return message;
}
+
+ public Map<String, List<String>> getFieldMap(String sourceType) {
+ return configurations.get(sourceType).getEnrichmentFieldMap();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
new file mode 100644
index 0000000..5839f39
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.bolt;
+
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Tuple;
+import org.apache.metron.Constants;
+import org.apache.metron.bolt.SplitBolt;
+import org.apache.metron.domain.Enrichment;
+import org.apache.metron.enrichment.utils.EnrichmentUtils;
+import org.apache.metron.topology.TopologyUtils;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.UnsupportedEncodingException;
+import java.util.*;
+
+public class EnrichmentSplitterBolt extends SplitBolt<JSONObject> {
+ protected static final Logger LOG = LoggerFactory.getLogger(EnrichmentSplitterBolt.class);
+ private List<Enrichment> enrichments;
+ protected String messageFieldName;
+ private transient JSONParser parser;
+
+
+ public EnrichmentSplitterBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
+
+ public EnrichmentSplitterBolt withEnrichments(List<Enrichment> enrichments) {
+ this.enrichments = enrichments;
+ return this;
+ }
+
+ public EnrichmentSplitterBolt withMessageFieldName(String messageFieldName) {
+ this.messageFieldName = messageFieldName;
+ return this;
+ }
+ @Override
+ public void prepare(Map map, TopologyContext topologyContext) {
+ parser = new JSONParser();
+ }
+ @Override
+ public String getKey(Tuple tuple, JSONObject message) {
+ String key = null;
+ try {
+ key = tuple.getStringByField("key");
+ }
+ catch(Throwable t) {
+ //swallowing this just in case.
+ }
+ if(key != null) {
+ return key;
+ }
+ else {
+ return UUID.randomUUID().toString();
+ }
+ }
+
+ @Override
+ public JSONObject generateMessage(Tuple tuple) {
+ JSONObject message = null;
+ if (messageFieldName == null) {
+ byte[] data = tuple.getBinary(0);
+ try {
+ message = (JSONObject) parser.parse(new String(data, "UTF8"));
+ } catch (ParseException | UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
+ } else {
+ message = (JSONObject) tuple.getValueByField(messageFieldName);
+ }
+ return message;
+ }
+
+ @Override
+ public Set<String> getStreamIds() {
+ Set<String> streamIds = new HashSet<>();
+ for(Enrichment enrichment: enrichments) {
+ streamIds.add(enrichment.getType());
+ }
+ return streamIds;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Map<String, JSONObject> splitMessage(JSONObject message) {
+ Map<String, JSONObject> streamMessageMap = new HashMap<>();
+ String sourceType = TopologyUtils.getSourceType(message);
+ Map<String, List<String>> enrichmentFieldMap = getFieldMap(sourceType);
+ for (String enrichmentType : enrichmentFieldMap.keySet()) {
+ List<String> fields = enrichmentFieldMap.get(enrichmentType);
+ JSONObject enrichmentObject = new JSONObject();
+ if (fields != null && fields.size() > 0) {
+ for (String field : fields) {
+ enrichmentObject.put(getKeyName(enrichmentType, field), message.get(field));
+ }
+ enrichmentObject.put(Constants.SOURCE_TYPE, sourceType);
+ streamMessageMap.put(enrichmentType, enrichmentObject);
+ }
+ }
+ return streamMessageMap;
+ }
+
+ protected Map<String, List<String>> getFieldMap(String sourceType) {
+ return configurations.get(sourceType).getEnrichmentFieldMap();
+ }
+
+ protected String getKeyName(String type, String field) {
+ return EnrichmentUtils.getEnrichmentKey(type, field);
+ }
+
+ @Override
+ public void declareOther(OutputFieldsDeclarer declarer) {
+
+ }
+
+ @Override
+ public void emitOther(Tuple tuple, JSONObject message) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
index 11ae1ef..b184975 100644
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
@@ -21,12 +21,13 @@ package org.apache.metron.enrichment.bolt;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import backtype.storm.topology.base.BaseRichBolt;
import com.google.common.base.Splitter;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.Iterables;
+import org.apache.metron.Constants;
+import org.apache.metron.bolt.ConfiguredBolt;
import org.apache.metron.domain.Enrichment;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.json.simple.JSONObject;
@@ -39,7 +40,7 @@ import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
-import org.apache.metron.helpers.topology.ErrorGenerator;
+import org.apache.metron.helpers.topology.ErrorUtils;
/**
* Uses an adapter to enrich telemetry messages with additional metadata
@@ -61,31 +62,32 @@ import org.apache.metron.helpers.topology.ErrorGenerator;
**/
@SuppressWarnings({"rawtypes", "serial"})
-public class GenericEnrichmentBolt extends BaseRichBolt {
+public class GenericEnrichmentBolt extends ConfiguredBolt {
private static final Logger LOG = LoggerFactory
.getLogger(GenericEnrichmentBolt.class);
private OutputCollector collector;
- protected String streamId;
- protected Enrichment<EnrichmentAdapter> enrichment;
+ protected String enrichmentType;
protected EnrichmentAdapter adapter;
protected transient CacheLoader<String, JSONObject> loader;
protected transient LoadingCache<String, JSONObject> cache;
protected Long maxCacheSize;
protected Long maxTimeRetain;
+ public GenericEnrichmentBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
/**
- * @param enrichment Object holding enrichment metadata
+ * @param enrichment enrichment
* @return Instance of this class
*/
- public GenericEnrichmentBolt withEnrichment(Enrichment<EnrichmentAdapter> enrichment) {
- this.streamId = enrichment.getName();
- this.enrichment = enrichment;
- this.adapter = this.enrichment.getAdapter();
+ public GenericEnrichmentBolt withEnrichment(Enrichment enrichment) {
+ this.enrichmentType = enrichment.getType();
+ this.adapter = enrichment.getAdapter();
return this;
}
@@ -113,17 +115,12 @@ public class GenericEnrichmentBolt extends BaseRichBolt {
public void prepare(Map conf, TopologyContext topologyContext,
OutputCollector collector) {
this.collector = collector;
- if (this.enrichment == null)
- throw new IllegalStateException("enrichment must be specified");
if (this.maxCacheSize == null)
throw new IllegalStateException("MAX_CACHE_SIZE_OBJECTS_NUM must be specified");
if (this.maxTimeRetain == null)
throw new IllegalStateException("MAX_TIME_RETAIN_MINUTES must be specified");
if (this.adapter == null)
throw new IllegalStateException("Adapter must be specified");
- if (this.enrichment.getFields() == null)
- throw new IllegalStateException(
- "Fields to be enriched must be specified");
loader = new CacheLoader<String, JSONObject>() {
public JSONObject load(String key) throws Exception {
return adapter.enrich(key);
@@ -141,7 +138,7 @@ public class GenericEnrichmentBolt extends BaseRichBolt {
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declareStream(streamId, new Fields("key", "message"));
+ declarer.declareStream(enrichmentType, new Fields("key", "message"));
declarer.declareStream("error", new Fields("message"));
}
@@ -156,26 +153,40 @@ public class GenericEnrichmentBolt extends BaseRichBolt {
throw new Exception("Could not parse binary stream to JSON");
if (key == null)
throw new Exception("Key is not valid");
- for (String field : enrichment.getFields()) {
- JSONObject enrichedField = new JSONObject();
+ for (Object o : rawMessage.keySet()) {
+ String field = (String) o;
String value = (String) rawMessage.get(field);
- if (value != null && value.length() != 0) {
- adapter.logAccess(value);
- enrichedField = cache.getUnchecked(value);
- if (enrichedField == null)
- throw new Exception("[Metron] Could not enrich string: "
- + value);
+ if (field.equals(Constants.SOURCE_TYPE)) {
+ enrichedMessage.put(Constants.SOURCE_TYPE, value);
+ } else {
+ JSONObject enrichedField = new JSONObject();
+ if (value != null && value.length() != 0) {
+ adapter.logAccess(value);
+ enrichedField = cache.getUnchecked(value);
+ if (enrichedField == null)
+ throw new Exception("[Metron] Could not enrich string: "
+ + value);
+ }
+ if (!enrichedField.isEmpty()) {
+ for (Object enrichedKey : enrichedField.keySet()) {
+ enrichedMessage.put(field + "." + enrichedKey, enrichedField.get(enrichedKey));
+ }
+ } else {
+ enrichedMessage.put(field, "");
+ }
+ if (enrichmentType.equals("host")) {
+ String test = "";
+ }
}
- enrichedMessage.put(Iterables.getLast(Splitter.on('/').split(field)), enrichedField);
}
if (!enrichedMessage.isEmpty()) {
- collector.emit(streamId, new Values(key, enrichedMessage));
+ collector.emit(enrichmentType, new Values(key, enrichedMessage));
}
} catch (Exception e) {
LOG.error("[Metron] Unable to enrich message: " + rawMessage, e);
- JSONObject error = ErrorGenerator.generateErrorMessage("Enrichment problem: " + rawMessage, e);
+ JSONObject error = ErrorUtils.generateErrorMessage("Enrichment problem: " + rawMessage, e);
if (key != null) {
- collector.emit(streamId, new Values(key, enrichedMessage));
+ collector.emit(enrichmentType, new Values(key, enrichedMessage));
}
collector.emit("error", new Values(error));
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
new file mode 100644
index 0000000..ba17fdb
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.bolt;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+public class ThreatIntelJoinBolt extends EnrichmentJoinBolt {
+
+ protected static final Logger LOG = LoggerFactory
+ .getLogger(ThreatIntelJoinBolt.class);
+
+ public ThreatIntelJoinBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
+
+ @Override
+ public Map<String, List<String>> getFieldMap(String sourceType) {
+ return configurations.get(sourceType).getThreatIntelFieldMap();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
new file mode 100644
index 0000000..a43360e
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.bolt;
+
+import org.apache.metron.enrichment.utils.ThreatIntelUtils;
+
+import java.util.List;
+import java.util.Map;
+
+public class ThreatIntelSplitterBolt extends EnrichmentSplitterBolt {
+
+ public ThreatIntelSplitterBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
+
+ @Override
+ protected Map<String, List<String>> getFieldMap(String sourceType) {
+ return configurations.get(sourceType).getThreatIntelFieldMap();
+ }
+
+ @Override
+ protected String getKeyName(String type, String field) {
+ return ThreatIntelUtils.getThreatIntelKey(type, field);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java
new file mode 100644
index 0000000..228f844
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/utils/EnrichmentUtils.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.utils;
+
+import com.google.common.base.Joiner;
+
+public class EnrichmentUtils {
+
+ public static final String KEY_PREFIX = "enrichments";
+
+ public static String getEnrichmentKey(String enrichmentName, String field) {
+ return Joiner.on(".").join(new String[]{KEY_PREFIX, enrichmentName, field});
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java
new file mode 100644
index 0000000..7898ccd
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/utils/ThreatIntelUtils.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.utils;
+
+import com.google.common.base.Joiner;
+
+public class ThreatIntelUtils {
+
+ public static final String KEY_PREFIX = "threatintels";
+
+ public static String getThreatIntelKey(String threatIntelName, String field) {
+ return Joiner.on(".").join(new String[]{KEY_PREFIX, threatIntelName, field});
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/AbstractIndexingBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/AbstractIndexingBolt.java b/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/AbstractIndexingBolt.java
index 3023953..423a5c2 100644
--- a/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/AbstractIndexingBolt.java
+++ b/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/AbstractIndexingBolt.java
@@ -21,6 +21,7 @@ package org.apache.metron.indexing;
import java.io.IOException;
import java.util.Map;
+import org.apache.metron.bolt.ConfiguredBolt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,7 +35,7 @@ import org.apache.metron.index.interfaces.IndexAdapter;
import org.apache.metron.metrics.MetricReporter;
@SuppressWarnings("rawtypes")
-public abstract class AbstractIndexingBolt extends BaseRichBolt {
+public abstract class AbstractIndexingBolt extends ConfiguredBolt {
/**
*
*/
@@ -56,6 +57,10 @@ public abstract class AbstractIndexingBolt extends BaseRichBolt {
protected Counter ackCounter, emitCounter, failCounter;
+ public AbstractIndexingBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
+
protected void registerCounters() {
String ackString = _adapter.getClass().getSimpleName() + ".ack";
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/TelemetryIndexingBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/TelemetryIndexingBolt.java b/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/TelemetryIndexingBolt.java
index eaeb1c6..ff151c7 100644
--- a/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/TelemetryIndexingBolt.java
+++ b/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/TelemetryIndexingBolt.java
@@ -34,7 +34,7 @@ import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
-import org.apache.metron.helpers.topology.ErrorGenerator;
+import org.apache.metron.helpers.topology.ErrorUtils;
import org.apache.metron.index.interfaces.IndexAdapter;
import org.apache.metron.json.serialization.JSONEncoderHelper;
import org.apache.metron.metrics.MetricReporter;
@@ -64,6 +64,10 @@ public class TelemetryIndexingBolt extends AbstractIndexingBolt {
private Set<Tuple> tuple_queue = new HashSet<Tuple>();
+ public TelemetryIndexingBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
+
/**
*
* @param IndexIP
@@ -89,17 +93,6 @@ public class TelemetryIndexingBolt extends AbstractIndexingBolt {
/**
*
- * @param IndexName
- * name of the index in ElasticSearch/Solr/etc...
- * @return instance of bolt
- */
- public TelemetryIndexingBolt withIndexName(String IndexName) {
- _IndexName = IndexName;
- return this;
- }
-
- /**
- *
* @param ClusterName
* name of cluster to index into in ElasticSearch/Solr/etc...
* @return instance of bolt
@@ -146,7 +139,7 @@ public class TelemetryIndexingBolt extends AbstractIndexingBolt {
/**
*
- * @param dateFormat
+ * @param indexTimestamp
* timestamp to append to index names
* @return instance of bolt
*/
@@ -185,7 +178,7 @@ public class TelemetryIndexingBolt extends AbstractIndexingBolt {
e.printStackTrace();
- JSONObject error = ErrorGenerator.generateErrorMessage(new String("bulk index problem"), e);
+ JSONObject error = ErrorUtils.generateErrorMessage(new String("bulk index problem"), e);
_collector.emit("error", new Values(error));
}
@@ -235,7 +228,7 @@ public class TelemetryIndexingBolt extends AbstractIndexingBolt {
failCounter.inc();
- JSONObject error = ErrorGenerator.generateErrorMessage(new String("bulk index problem"), e);
+ JSONObject error = ErrorUtils.generateErrorMessage(new String("bulk index problem"), e);
_collector.emit("error", new Values(error));
}
tuple_queue.clear();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/adapters/ESTimedRotatingAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/adapters/ESTimedRotatingAdapter.java b/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/adapters/ESTimedRotatingAdapter.java
index b1a9ca4..fd4c067 100644
--- a/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/adapters/ESTimedRotatingAdapter.java
+++ b/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/indexing/adapters/ESTimedRotatingAdapter.java
@@ -180,7 +180,8 @@ public class ESTimedRotatingAdapter extends AbstractIndexAdapter implements
r.getResponse();
_LOG.trace("[Metron] ES SUCCESS MESSAGE: " + r.getFailureMessage());
}
-
+
+
bulk_set.clear();
if (resp.hasFailures()) {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/writer/ElasticSearchWriter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/writer/ElasticSearchWriter.java b/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/writer/ElasticSearchWriter.java
new file mode 100644
index 0000000..a0df685
--- /dev/null
+++ b/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/writer/ElasticSearchWriter.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer;
+
+import backtype.storm.tuple.Tuple;
+import org.apache.metron.domain.SourceConfig;
+import org.apache.metron.writer.interfaces.BulkMessageWriter;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.json.simple.JSONObject;
+
+import java.io.Serializable;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+public class ElasticsearchWriter implements BulkMessageWriter<JSONObject>, Serializable {
+
+ private String clusterName;
+ private Map<String, String> optionalSettings;
+ private transient TransportClient client;
+ private String host;
+ private int port;
+ private SimpleDateFormat dateFormat;
+
+ public ElasticsearchWriter(String clusterName, String host, int port, String dateFormat) {
+ this.clusterName = clusterName;
+ this.host = host;
+ this.port = port;
+ this.dateFormat = new SimpleDateFormat(dateFormat);
+ }
+
+ public ElasticsearchWriter withOptionalSettings(Map<String, String> optionalSettings) {
+ this.optionalSettings = optionalSettings;
+ return this;
+ }
+
+ @Override
+ public void init() {
+ ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder();
+ builder.put("cluster.name", clusterName);
+ builder.put("client.transport.ping_timeout","500s");
+ if (optionalSettings != null) {
+ builder.put(optionalSettings);
+ }
+ client = new TransportClient(builder.build())
+ .addTransportAddress(new InetSocketTransportAddress(host, port));
+
+ }
+
+ @Override
+ public void write(String sourceType, SourceConfig configuration, List<Tuple> tuples, List<JSONObject> messages) throws Exception {
+ String indexPostfix = dateFormat.format(new Date());
+ BulkRequestBuilder bulkRequest = client.prepareBulk();
+ for(JSONObject message: messages) {
+ String indexName = sourceType;
+ if (configuration != null) {
+ indexName = configuration.getIndex();
+ }
+ IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName + "_" + indexPostfix,
+ sourceType);
+ indexRequestBuilder.setSource(message.toJSONString());
+ bulkRequest.add(indexRequestBuilder);
+ }
+ BulkResponse resp = bulkRequest.execute().actionGet();
+ if (resp.hasFailures()) {
+ throw new Exception(resp.buildFailureMessage());
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ client.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/writer/HdfsWriter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/writer/HdfsWriter.java b/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/writer/HdfsWriter.java
new file mode 100644
index 0000000..eace952
--- /dev/null
+++ b/metron-streaming/Metron-Indexing/src/main/java/org/apache/metron/writer/HdfsWriter.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer;
+
+import backtype.storm.tuple.Tuple;
+import org.apache.metron.domain.SourceConfig;
+import org.apache.metron.writer.interfaces.BulkMessageWriter;
+import org.json.simple.JSONObject;
+
+import java.io.Serializable;
+import java.util.List;
+
+public class HdfsWriter implements BulkMessageWriter<JSONObject>, Serializable {
+
+ @Override
+ public void init() {
+
+ }
+
+ @Override
+ public void write(String sourceType, SourceConfig configuration, List<Tuple> tuples, List<JSONObject> messages) throws Exception {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-MessageParsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/pom.xml b/metron-streaming/Metron-MessageParsers/pom.xml
index 971b9d0..a697aa8 100644
--- a/metron-streaming/Metron-MessageParsers/pom.xml
+++ b/metron-streaming/Metron-MessageParsers/pom.xml
@@ -68,6 +68,19 @@
<artifactId>grok</artifactId>
<version>0.1.0</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.9.2</artifactId>
+ <version>${global_kafka_version}</version>
+ <exclusions>
+ <!--exclusion> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId>
+ </exclusion -->
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<reporting>
<plugins>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/bolt/ParserBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/bolt/ParserBolt.java b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/bolt/ParserBolt.java
new file mode 100644
index 0000000..27294ef
--- /dev/null
+++ b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/bolt/ParserBolt.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.bolt;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Tuple;
+import org.apache.metron.Constants;
+import org.apache.metron.filters.GenericMessageFilter;
+import org.apache.metron.helpers.topology.ErrorUtils;
+import org.apache.metron.parser.interfaces.MessageFilter;
+import org.apache.metron.parser.interfaces.MessageParser;
+import org.apache.metron.writer.interfaces.MessageWriter;
+import org.json.simple.JSONObject;
+
+import java.util.List;
+import java.util.Map;
+
+public class ParserBolt extends ConfiguredBolt {
+
+ private OutputCollector collector;
+ private MessageParser<JSONObject> parser;
+ private MessageFilter<JSONObject> filter = new GenericMessageFilter();
+ private MessageWriter<JSONObject> writer;
+ private String sourceType;
+
+ public ParserBolt(String zookeeperUrl, String sourceType, MessageParser<JSONObject> parser, MessageWriter<JSONObject> writer) {
+ super(zookeeperUrl);
+ this.parser = parser;
+ this.sourceType = sourceType;
+ this.writer = writer;
+ }
+
+ public ParserBolt withMessageFilter(MessageFilter<JSONObject> filter) {
+ this.filter = filter;
+ return this;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ super.prepare(stormConf, context, collector);
+ this.collector = collector;
+ parser.init();
+ writer.init();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void execute(Tuple tuple) {
+ byte[] originalMessage = tuple.getBinary(0);
+ try {
+ List<JSONObject> messages = parser.parse(originalMessage);
+ for(JSONObject message: messages) {
+ if (parser.validate(message)) {
+ if (filter != null && filter.emitTuple(message)) {
+ message.put(Constants.SOURCE_TYPE, sourceType);
+ writer.write(sourceType, configurations.get(sourceType), tuple, message);
+ }
+ }
+ }
+ collector.ack(tuple);
+ } catch (Throwable ex) {
+ ErrorUtils.handleError(collector, ex, Constants.ERROR_STREAM);
+ }
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/bolt/PcapParserBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/bolt/PcapParserBolt.java b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/bolt/PcapParserBolt.java
index ab14f54..05582e5 100644
--- a/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/bolt/PcapParserBolt.java
+++ b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/bolt/PcapParserBolt.java
@@ -27,18 +27,22 @@ import java.util.List;
public class PcapParserBolt extends TelemetryParserBolt {
+ public PcapParserBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
+
@Override
public void declareOther(OutputFieldsDeclarer declarer) {
declarer.declareStream("raw", new Fields("key", "value", "timestamp") );
}
@Override
- public void emitOther(Tuple tuple, List<JSONObject> messages) {
- for(JSONObject message: messages) {
+ public void emitOther(Tuple tuple, JSONObject message) {
+ //for(JSONObject message: messages) {
String key = (String) message.get("pcap_id");
long timestamp = (long) message.get("ts_micro");
collector.emit("raw", tuple, new Values(key, tuple.getBinary(0),
timestamp));
- }
+ //}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/bolt/TelemetryParserBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/bolt/TelemetryParserBolt.java b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/bolt/TelemetryParserBolt.java
index c53ba8e..e236760 100644
--- a/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/bolt/TelemetryParserBolt.java
+++ b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/bolt/TelemetryParserBolt.java
@@ -18,11 +18,8 @@
package org.apache.metron.bolt;
import backtype.storm.task.TopologyContext;
-import backtype.storm.tuple.Tuple;
-import backtype.storm.tuple.Values;
-import org.apache.metron.enrichment.EnrichmentSplitterBolt;
+import org.apache.metron.enrichment.bolt.EnrichmentSplitterBolt;
import org.apache.metron.filters.GenericMessageFilter;
-import org.apache.metron.helpers.topology.ErrorGenerator;
import org.apache.metron.parser.interfaces.MessageFilter;
import org.apache.metron.parser.interfaces.MessageParser;
import org.json.simple.JSONObject;
@@ -39,6 +36,10 @@ public class TelemetryParserBolt extends EnrichmentSplitterBolt {
protected MessageParser<JSONObject> parser;
protected MessageFilter<JSONObject> filter = new GenericMessageFilter();
+ public TelemetryParserBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
+
/**
* @param parser The parser class for parsing the incoming raw message byte
* stream
@@ -74,6 +75,7 @@ public class TelemetryParserBolt extends EnrichmentSplitterBolt {
+ /*
@Override
public List<JSONObject> generateMessages(Tuple tuple) {
List<JSONObject> filteredMessages = new ArrayList<>();
@@ -102,7 +104,7 @@ public class TelemetryParserBolt extends EnrichmentSplitterBolt {
collector.emit("error", new Values(error));
}
return filteredMessages;
- }
+ }*/
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parser/MessageParser.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parser/MessageParser.java b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parser/MessageParser.java
new file mode 100644
index 0000000..ca52fd8
--- /dev/null
+++ b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parser/MessageParser.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parser;
+
+public interface MessageParser<T> {
+
+ void init();
+ T parse(byte[] rawMessage);
+ boolean validate(T message);
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/GrokParser.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/GrokParser.java b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/GrokParser.java
new file mode 100644
index 0000000..9c7e6af
--- /dev/null
+++ b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/GrokParser.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsing.parsers;
+
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import oi.thekraken.grok.api.exception.GrokException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.parser.interfaces.MessageParser;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Serializable;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
+
+public class GrokParser implements MessageParser<JSONObject>, Serializable {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(GrokParser.class);
+
+ private transient Grok grok;
+ private String grokHdfsPath;
+ private String patternLabel;
+ private String[] timeFields = new String[0];
+ private String timestampField;
+ private String dateFormat = "yyyy-MM-dd HH:mm:ss.S z";
+ private TimeZone timeZone = TimeZone.getTimeZone("UTC");
+
+ public GrokParser(String grokHdfsPath, String patterLabel) {
+ this.grokHdfsPath = grokHdfsPath;
+ this.patternLabel = patterLabel;
+ }
+
+ public GrokParser withTimestampField(String timestampField) {
+ this.timestampField = timestampField;
+ return this;
+ }
+
+ public GrokParser withTimeFields(String... timeFields) {
+ this.timeFields = timeFields;
+ return this;
+ }
+
+ public GrokParser withDateFormat(String dateFormat) {
+ this.dateFormat = dateFormat;
+ return this;
+ }
+
+ public GrokParser withTimeZone(String timeZone) {
+ this.timeZone = TimeZone.getTimeZone(timeZone);
+ return this;
+ }
+
+ @Override
+ public void init() {
+ grok = new Grok();
+ try {
+ InputStream commonInputStream = getClass().getResourceAsStream
+ ("/patterns/common");
+ grok.addPatternFromReader(new InputStreamReader(commonInputStream));
+ InputStream patterInputStream = FileSystem.get(new Configuration()).open(new
+ Path(grokHdfsPath));
+ grok.addPatternFromReader(new InputStreamReader(patterInputStream));
+ grok.compile("%{" + patternLabel + "}");
+ } catch (GrokException e) {
+ LOG.error(e.getMessage(), e);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<JSONObject> parse(byte[] rawMessage) {
+ if (grok == null) init();
+ List<JSONObject> messages = new ArrayList<>();
+ try {
+ String originalMessage = new String(rawMessage, "UTF-8");
+ Match gm = grok.match(originalMessage);
+ gm.captures();
+ JSONObject message = new JSONObject();
+ message.putAll(gm.toMap());
+ message.put("original_string", originalMessage);
+ for(String timeField: timeFields) {
+ String fieldValue = (String) message.get(timeField);
+ if (fieldValue != null) {
+ message.put(timeField, toEpoch(fieldValue));
+ }
+ }
+ if (timestampField != null) {
+ message.put("timestamp", message.get(timestampField));
+ }
+ message.remove(patternLabel);
+ messages.add(message);
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ return null;
+ }
+ return messages;
+ }
+
+ @Override
+ public boolean validate(JSONObject message) {
+ Object timestampObject = message.get("timestamp");
+ if (timestampObject instanceof Long) {
+ Long timestamp = (Long) timestampObject;
+ if (timestamp > 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private long toEpoch(String datetime) throws ParseException {
+ SimpleDateFormat sdf = new SimpleDateFormat(dateFormat);
+ sdf.setTimeZone(timeZone);
+ Date date = sdf.parse(datetime);
+ return date.getTime();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/PcapParser.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/PcapParser.java b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/PcapParser.java
index e8c0bc9..c5677f3 100644
--- a/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/PcapParser.java
+++ b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/PcapParser.java
@@ -17,18 +17,13 @@
*/
package org.apache.metron.parsing.parsers;
-import java.io.EOFException;
-import java.io.File;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-import backtype.storm.tuple.Tuple;
-import backtype.storm.tuple.Values;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.apache.metron.parser.interfaces.MessageParser;
+import org.apache.metron.pcap.Constants;
+import org.apache.metron.pcap.MetronEthernetDecoder;
+import org.apache.metron.pcap.PacketInfo;
+import org.apache.metron.pcap.PcapByteInputStream;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.krakenapps.pcap.decoder.ethernet.EthernetDecoder;
@@ -42,10 +37,12 @@ import org.krakenapps.pcap.packet.PacketHeader;
import org.krakenapps.pcap.packet.PcapPacket;
import org.krakenapps.pcap.util.Buffer;
-import org.apache.metron.pcap.Constants;
-import org.apache.metron.pcap.MetronEthernetDecoder;
-import org.apache.metron.pcap.PacketInfo;
-import org.apache.metron.pcap.PcapByteInputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
public class PcapParser implements MessageParser<JSONObject>, Serializable {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/writer/KafkaWriter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/writer/KafkaWriter.java b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/writer/KafkaWriter.java
new file mode 100644
index 0000000..8372e14
--- /dev/null
+++ b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/writer/KafkaWriter.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer;
+
+import backtype.storm.tuple.Tuple;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.metron.Constants;
+import org.apache.metron.domain.SourceConfig;
+import org.apache.metron.writer.interfaces.MessageWriter;
+import org.json.simple.JSONObject;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+public class KafkaWriter implements MessageWriter<JSONObject>, Serializable {
+
+ private String brokerUrl;
+ private String keySerializer = "org.apache.kafka.common.serialization.StringSerializer";
+ private String valueSerializer = "org.apache.kafka.common.serialization.StringSerializer";
+ private int requiredAcks = 1;
+ private KafkaProducer kafkaProducer;
+
+ public KafkaWriter(String brokerUrl) {
+ this.brokerUrl = brokerUrl;
+ }
+
+ public KafkaWriter withKeySerializer(String keySerializer) {
+ this.keySerializer = keySerializer;
+ return this;
+ }
+
+ public KafkaWriter withValueSerializer(String valueSerializer) {
+ this.valueSerializer = valueSerializer;
+ return this;
+ }
+
+ public KafkaWriter withRequiredAcks(int requiredAcks) {
+ this.requiredAcks = requiredAcks;
+ return this;
+ }
+
+ @Override
+ public void init() {
+ Map<String, Object> producerConfig = new HashMap<>();
+ producerConfig.put("bootstrap.servers", brokerUrl);
+ producerConfig.put("key.serializer", keySerializer);
+ producerConfig.put("value.serializer", valueSerializer);
+ producerConfig.put("request.required.acks", requiredAcks);
+ this.kafkaProducer = new KafkaProducer<>(producerConfig);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void write(String sourceType, SourceConfig configuration, Tuple tuple, JSONObject message) throws Exception {
+ kafkaProducer.send(new ProducerRecord<String, String>(Constants.ENRICHMENT_TOPIC, message.toJSONString()));
+ }
+
+ @Override
+ public void close() throws Exception {
+ kafkaProducer.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-MessageParsers/src/main/resources/patterns/common
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/main/resources/patterns/common b/metron-streaming/Metron-MessageParsers/src/main/resources/patterns/common
new file mode 100644
index 0000000..10c72dc
--- /dev/null
+++ b/metron-streaming/Metron-MessageParsers/src/main/resources/patterns/common
@@ -0,0 +1,96 @@
+# Forked from https://github.com/elasticsearch/logstash/tree/v1.4.0/patterns
+
+USERNAME [a-zA-Z0-9._-]+
+USER %{USERNAME:UNWANTED}
+INT (?:[+-]?(?:[0-9]+))
+BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
+NUMBER (?:%{BASE10NUM:UNWANTED})
+BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
+BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
+
+POSINT \b(?:[1-9][0-9]*)\b
+NONNEGINT \b(?:[0-9]+)\b
+WORD \b\w+\b
+NOTSPACE \S+
+SPACE \s*
+DATA .*?
+GREEDYDATA .*
+#QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`)))
+QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
+UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
+
+# Networking
+MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED})
+CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
+WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
+COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
+IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5
]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
+IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
+IP (?:%{IPV6:UNWANTED}|%{IPV4:UNWANTED})
+HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
+HOST %{HOSTNAME:UNWANTED}
+IPORHOST (?:%{HOSTNAME:UNWANTED}|%{IP:UNWANTED})
+HOSTPORT (?:%{IPORHOST}:%{POSINT:PORT})
+
+# paths
+PATH (?:%{UNIXPATH}|%{WINPATH})
+UNIXPATH (?>/(?>[\w_%!$@:.,~-]+|\\.)*)+
+#UNIXPATH (?<![\w\/])(?:/[^\/\s?*]*)+
+TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
+WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
+URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
+URIHOST %{IPORHOST}(?::%{POSINT:port})?
+# uripath comes loosely from RFC1738, but mostly from what Firefox
+# doesn't turn into %XX
+URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
+#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
+URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
+URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
+URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
+
+# Months: January, Feb, 3, 03, 12, December
+MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
+MONTHNUM (?:0?[1-9]|1[0-2])
+MONTHNUM2 (?:0[1-9]|1[0-2])
+MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
+
+# Days: Monday, Tue, Thu, etc...
+DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
+
+# Years?
+YEAR (?>\d\d){1,2}
+# Time: HH:MM:SS
+#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?
+# I'm still on the fence about using grok to perform the time match,
+# since it's probably slower.
+# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)?
+HOUR (?:2[0123]|[01]?[0-9])
+MINUTE (?:[0-5][0-9])
+# '60' is a leap second in most time standards and thus is valid.
+SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
+TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
+# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
+DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
+DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
+ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
+ISO8601_SECOND (?:%{SECOND}|60)
+TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
+DATE %{DATE_US}|%{DATE_EU}
+DATESTAMP %{DATE}[- ]%{TIME}
+TZ (?:[PMCE][SD]T|UTC)
+DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
+DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
+DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
+DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
+GREEDYDATA .*
+
+# Syslog Dates: Month Day HH:MM:SS
+SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
+PROG (?:[\w._/%-]+)
+SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
+SYSLOGHOST %{IPORHOST}
+SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
+HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
+
+# Shortcuts
+QS %{QUOTEDSTRING:UNWANTED}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-MessageParsers/src/main/resources/patterns/yaf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/main/resources/patterns/yaf b/metron-streaming/Metron-MessageParsers/src/main/resources/patterns/yaf
index 3ac640e..8fc130e 100644
--- a/metron-streaming/Metron-MessageParsers/src/main/resources/patterns/yaf
+++ b/metron-streaming/Metron-MessageParsers/src/main/resources/patterns/yaf
@@ -1,113 +1,2 @@
-# Forked from https://github.com/elasticsearch/logstash/tree/v1.4.0/patterns
-
-USERNAME [a-zA-Z0-9._-]+
-USER %{USERNAME:UNWANTED}
-INT (?:[+-]?(?:[0-9]+))
-BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
-NUMBER (?:%{BASE10NUM:UNWANTED})
-BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
-BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
-
-POSINT \b(?:[1-9][0-9]*)\b
-NONNEGINT \b(?:[0-9]+)\b
-WORD \b\w+\b
-NOTSPACE \S+
-SPACE \s*
-DATA .*?
-GREEDYDATA .*
-#QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`)))
-QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
-UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
-
-# Networking
-MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED})
-CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
-WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
-COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
-IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5
]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
-IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
-IP (?:%{IPV6:UNWANTED}|%{IPV4:UNWANTED})
-HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
-HOST %{HOSTNAME:UNWANTED}
-IPORHOST (?:%{HOSTNAME:UNWANTED}|%{IP:UNWANTED})
-HOSTPORT (?:%{IPORHOST}:%{POSINT:PORT})
-
-# paths
-PATH (?:%{UNIXPATH}|%{WINPATH})
-UNIXPATH (?>/(?>[\w_%!$@:.,~-]+|\\.)*)+
-#UNIXPATH (?<![\w\/])(?:/[^\/\s?*]*)+
-TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
-WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
-URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
-URIHOST %{IPORHOST}(?::%{POSINT:port})?
-# uripath comes loosely from RFC1738, but mostly from what Firefox
-# doesn't turn into %XX
-URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
-#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
-URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
-URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
-URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
-
-# Months: January, Feb, 3, 03, 12, December
-MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
-MONTHNUM (?:0?[1-9]|1[0-2])
-MONTHNUM2 (?:0[1-9]|1[0-2])
-MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
-
-# Days: Monday, Tue, Thu, etc...
-DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
-
-# Years?
-YEAR (?>\d\d){1,2}
-# Time: HH:MM:SS
-#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?
-# I'm still on the fence about using grok to perform the time match,
-# since it's probably slower.
-# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)?
-HOUR (?:2[0123]|[01]?[0-9])
-MINUTE (?:[0-5][0-9])
-# '60' is a leap second in most time standards and thus is valid.
-SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
-TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
-# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
-DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
-DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
-ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
-ISO8601_SECOND (?:%{SECOND}|60)
-TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
-DATE %{DATE_US}|%{DATE_EU}
-DATESTAMP %{DATE}[- ]%{TIME}
-TZ (?:[PMCE][SD]T|UTC)
-DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
-DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
-DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
-DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
-GREEDYDATA .*
-
-# Syslog Dates: Month Day HH:MM:SS
-SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
-PROG (?:[\w._/%-]+)
-SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
-SYSLOGHOST %{IPORHOST}
-SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
-HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
-
-# Shortcuts
-QS %{QUOTEDSTRING:UNWANTED}
-
-# Log formats
-SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
-
-MESSAGESLOG %{SYSLOGBASE} %{DATA}
-
-COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
-COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
-
-# Log Levels
-LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
-
-# Yaf
YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T ]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED}
-YAF_DELIMITED %{YAF_TIME_FORMAT:start_time}\|%{YAF_TIME_FORMAT:end_time}\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\|%{SPACE:UNWANTED}%{INT:proto}\|%{SPACE:UNWANTED}%{IP:sip}\|%{SPACE:UNWANTED}%{INT:sp}\|%{SPACE:UNWANTED}%{IP:dip}\|%{SPACE:UNWANTED}%{INT:dp}\|%{SPACE:UNWANTED}%{DATA:iflags}\|%{SPACE:UNWANTED}%{DATA:uflags}\|%{SPACE:UNWANTED}%{DATA:riflags}\|%{SPACE:UNWANTED}%{DATA:ruflags}\|%{SPACE:UNWANTED}%{WORD:isn}\|%{SPACE:UNWANTED}%{DATA:risn}\|%{SPACE:UNWANTED}%{DATA:tag}\|%{GREEDYDATA:rtag}\|%{SPACE:UNWANTED}%{INT:pkt}\|%{SPACE:UNWANTED}%{INT:oct}\|%{SPACE:UNWANTED}%{INT:rpkt}\|%{SPACE:UNWANTED}%{INT:roct}\|%{SPACE:UNWANTED}%{INT:app}\|%{GREEDYDATA:end_reason}
-
-
+YAF_DELIMITED %{YAF_TIME_FORMAT:start_time}\|%{YAF_TIME_FORMAT:end_time}\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\|%{SPACE:UNWANTED}%{INT:proto}\|%{SPACE:UNWANTED}%{IP:sip}\|%{SPACE:UNWANTED}%{INT:sp}\|%{SPACE:UNWANTED}%{IP:dip}\|%{SPACE:UNWANTED}%{INT:dp}\|%{SPACE:UNWANTED}%{DATA:iflags}\|%{SPACE:UNWANTED}%{DATA:uflags}\|%{SPACE:UNWANTED}%{DATA:riflags}\|%{SPACE:UNWANTED}%{DATA:ruflags}\|%{SPACE:UNWANTED}%{WORD:isn}\|%{SPACE:UNWANTED}%{DATA:risn}\|%{SPACE:UNWANTED}%{DATA:tag}\|%{GREEDYDATA:rtag}\|%{SPACE:UNWANTED}%{INT:pkt}\|%{SPACE:UNWANTED}%{INT:oct}\|%{SPACE:UNWANTED}%{INT:rpkt}\|%{SPACE:UNWANTED}%{INT:roct}\|%{SPACE:UNWANTED}%{INT:app}\|%{GREEDYDATA:end_reason}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Testing/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/pom.xml b/metron-streaming/Metron-Testing/pom.xml
index d68d81d..5f1c946 100644
--- a/metron-streaming/Metron-Testing/pom.xml
+++ b/metron-streaming/Metron-Testing/pom.xml
@@ -78,7 +78,33 @@
</exclusion>
</exclusions>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.9.2</artifactId>
+ <version>${global_kafka_version}</version>
+ <classifier>test</classifier>
+ <exclusions>
+ <!--exclusion> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId>
+ </exclusion -->
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.9.2</artifactId>
+ <version>${global_kafka_version}</version>
+ <exclusions>
+ <!--exclusion> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId>
+ </exclusion -->
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/ComponentRunner.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/ComponentRunner.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/ComponentRunner.java
index 3e5e793..f9a8ca2 100644
--- a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/ComponentRunner.java
+++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/ComponentRunner.java
@@ -26,6 +26,7 @@ public class ComponentRunner {
LinkedHashMap<String, InMemoryComponent> components;
String[] startupOrder;
String[] shutdownOrder;
+ long timeBetweenAttempts;
public Builder() {
components = new LinkedHashMap<String, InMemoryComponent>();
}
@@ -43,6 +44,10 @@ public class ComponentRunner {
this.shutdownOrder = shutdownOrder;
return this;
}
+ public Builder withTimeBetweenAttempts(long timeBetweenAttempts) {
+ this.timeBetweenAttempts = timeBetweenAttempts;
+ return this;
+ }
private static String[] toOrderedList(Map<String, InMemoryComponent> components) {
String[] ret = new String[components.size()];
int i = 0;
@@ -58,7 +63,7 @@ public class ComponentRunner {
if(startupOrder == null) {
startupOrder = toOrderedList(components);
}
- return new ComponentRunner(components, startupOrder, shutdownOrder);
+ return new ComponentRunner(components, startupOrder, shutdownOrder, timeBetweenAttempts);
}
}
@@ -66,15 +71,17 @@ public class ComponentRunner {
LinkedHashMap<String, InMemoryComponent> components;
String[] startupOrder;
String[] shutdownOrder;
+ long timeBetweenAttempts;
public ComponentRunner( LinkedHashMap<String, InMemoryComponent> components
, String[] startupOrder
, String[] shutdownOrder
+ , long timeBetweenAttempts
)
{
this.components = components;
this.startupOrder = startupOrder;
this.shutdownOrder = shutdownOrder;
-
+ this.timeBetweenAttempts = timeBetweenAttempts;
}
public <T extends InMemoryComponent> T getComponent(String name, Class<T> clazz) {
@@ -97,10 +104,10 @@ public class ComponentRunner {
}
public <T> T process(Processor<T> successState) {
- return process(successState, 5, 30000, 120000);
+ return process(successState, 5, 120000);
}
- public <T> T process(Processor<T> successState, int numRetries, long timeBetweenAttempts, long maxTimeMs) {
+ public <T> T process(Processor<T> successState, int numRetries, long maxTimeMs) {
int retryCount = 0;
long start = System.currentTimeMillis();
while(true) {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/ElasticSearchComponent.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/ElasticSearchComponent.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/ElasticSearchComponent.java
index a7991c0..42d7a08 100644
--- a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/ElasticSearchComponent.java
+++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/ElasticSearchComponent.java
@@ -145,13 +145,13 @@ public class ElasticSearchComponent implements InMemoryComponent {
}
}
- public List<Map<String, Object>> getAllIndexedDocs(String index) throws IOException {
- return getAllIndexedDocs(index, "message");
+ public List<Map<String, Object>> getAllIndexedDocs(String index, String sourceType) throws IOException {
+ return getAllIndexedDocs(index, sourceType, null);
}
- public List<Map<String, Object>> getAllIndexedDocs(String index, String subMessage) throws IOException {
+ public List<Map<String, Object>> getAllIndexedDocs(String index, String sourceType, String subMessage) throws IOException {
getClient().admin().indices().refresh(new RefreshRequest());
SearchResponse response = getClient().prepareSearch(index)
- .setTypes("pcap_doc")
+ .setTypes(sourceType)
.setSource("message")
.setFrom(0)
.setSize(1000)
[6/9] incubator-metron git commit: METRON-56 Create unified
enrichment topology (merrimanr via cestella) closes
apache/incubator-metron#33
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/KafkaWithZKComponent.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/KafkaWithZKComponent.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/KafkaWithZKComponent.java
new file mode 100644
index 0000000..83ecd42
--- /dev/null
+++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/KafkaWithZKComponent.java
@@ -0,0 +1,228 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration.util.integration.components;
+
+
+import com.google.common.base.Function;
+import kafka.Kafka;
+import kafka.admin.AdminUtils;
+import kafka.api.FetchRequest;
+import kafka.api.FetchRequestBuilder;
+import kafka.consumer.ConsumerConfig;
+import kafka.consumer.ConsumerIterator;
+import kafka.consumer.KafkaStream;
+import kafka.javaapi.FetchResponse;
+import kafka.javaapi.consumer.ConsumerConnector;
+import kafka.javaapi.consumer.SimpleConsumer;
+import kafka.message.MessageAndOffset;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import kafka.server.KafkaConfig;
+import kafka.server.KafkaServer;
+import kafka.utils.*;
+import kafka.zk.EmbeddedZookeeper;
+import org.I0Itec.zkclient.ZkClient;
+import org.apache.metron.integration.util.integration.InMemoryComponent;
+import org.apache.zookeeper.KeeperException;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+
+public class KafkaWithZKComponent implements InMemoryComponent {
+
+
+ public static class Topic {
+ public int numPartitions;
+ public String name;
+
+ public Topic(String name, int numPartitions) {
+ this.numPartitions = numPartitions;
+ this.name = name;
+ }
+ }
+ private transient KafkaServer kafkaServer;
+ private transient EmbeddedZookeeper zkServer;
+ private transient ZkClient zkClient;
+ private transient ConsumerConnector consumer;
+ private String zookeeperConnectString;
+ private int brokerPort = 6667;
+ private List<Topic> topics = Collections.emptyList();
+ private Function<KafkaWithZKComponent, Void> postStartCallback;
+
+ public KafkaWithZKComponent withPostStartCallback(Function<KafkaWithZKComponent, Void> f) {
+ postStartCallback = f;
+ return this;
+ }
+
+ public KafkaWithZKComponent withExistingZookeeper(String zookeeperConnectString) {
+ this.zookeeperConnectString = zookeeperConnectString;
+ return this;
+ }
+
+ public KafkaWithZKComponent withBrokerPort(int brokerPort) {
+ if(brokerPort <= 0)
+ {
+ brokerPort = TestUtils.choosePort();
+ }
+ this.brokerPort = brokerPort;
+ return this;
+ }
+
+ public KafkaWithZKComponent withTopics(List<Topic> topics) {
+ this.topics = topics;
+ return this;
+ }
+
+ public List<Topic> getTopics() {
+ return topics;
+ }
+
+ public int getBrokerPort() {
+ return brokerPort;
+ }
+
+
+ public String getBrokerList() {
+ return "localhost:" + brokerPort;
+ }
+
+ public KafkaProducer<String, byte[]> createProducer()
+ {
+ return createProducer(new HashMap<String, Object>());
+ }
+
+ public KafkaProducer<String, byte[]> createProducer(Map<String, Object> properties)
+ {
+ Map<String, Object> producerConfig = new HashMap<>();
+ producerConfig.put("bootstrap.servers", getBrokerList());
+ producerConfig.put("key.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
+ producerConfig.put("value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
+ producerConfig.put("request.required.acks", "-1");
+ producerConfig.put("fetch.message.max.bytes", ""+ 1024*1024*10);
+ producerConfig.put("replica.fetch.max.bytes", "" + 1024*1024*10);
+ producerConfig.put("message.max.bytes", "" + 1024*1024*10);
+ producerConfig.put("message.send.max.retries", "10");
+ producerConfig.putAll(properties);
+ return new KafkaProducer<>(producerConfig);
+ }
+
+ @Override
+ public void start() {
+ // setup Zookeeper
+ if(zookeeperConnectString == null) {
+ String zkConnect = TestZKUtils.zookeeperConnect();
+ zkServer = new EmbeddedZookeeper(zkConnect);
+ zookeeperConnectString = zkServer.connectString();
+ }
+ zkClient = new ZkClient(zookeeperConnectString, 30000, 30000, ZKStringSerializer$.MODULE$);
+
+ // setup Broker
+ Properties props = TestUtils.createBrokerConfig(0, brokerPort, true);
+ KafkaConfig config = new KafkaConfig(props);
+ Time mock = new MockTime();
+ kafkaServer = TestUtils.createServer(config, mock);
+ for(Topic topic : getTopics()) {
+ try {
+ createTopic(topic.name, topic.numPartitions, true);
+ } catch (InterruptedException e) {
+ throw new RuntimeException("Unable to create topic", e);
+ }
+ }
+ postStartCallback.apply(this);
+ }
+
+ public String getZookeeperConnect() {
+ return zookeeperConnectString;
+ }
+
+ @Override
+ public void stop() {
+ kafkaServer.shutdown();
+ zkClient.close();
+ if(zkServer != null) {
+ zkServer.shutdown();
+ }
+
+ }
+
+ public List<byte[]> readMessages(String topic) {
+ SimpleConsumer consumer = new SimpleConsumer("localhost", 6667, 100000, 64 * 1024, "consumer");
+ FetchRequest req = new FetchRequestBuilder()
+ .clientId("consumer")
+ .addFetch(topic, 0, 0, 100000)
+ .build();
+ FetchResponse fetchResponse = consumer.fetch(req);
+ Iterator<MessageAndOffset> results = fetchResponse.messageSet(topic, 0).iterator();
+ List<byte[]> messages = new ArrayList<>();
+ while(results.hasNext()) {
+ ByteBuffer payload = results.next().message().payload();
+ byte[] bytes = new byte[payload.limit()];
+ payload.get(bytes);
+ messages.add(bytes);
+ }
+ return messages;
+ }
+
+ public ConsumerIterator<byte[], byte[]> getStreamIterator(String topic) {
+ return getStreamIterator(topic, "group0", "consumer0");
+ }
+ public ConsumerIterator<byte[], byte[]> getStreamIterator(String topic, String group, String consumerName) {
+ // setup simple consumer
+ Properties consumerProperties = TestUtils.createConsumerProperties(zkServer.connectString(), group, consumerName, -1);
+ consumer = kafka.consumer.Consumer.createJavaConsumerConnector(new ConsumerConfig(consumerProperties));
+ Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
+ topicCountMap.put(topic, 1);
+ Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);
+ KafkaStream<byte[], byte[]> stream = consumerMap.get(topic).get(0);
+ ConsumerIterator<byte[], byte[]> iterator = stream.iterator();
+ return iterator;
+ }
+
+ public void shutdownConsumer() {
+ consumer.shutdown();
+ }
+
+ public void createTopic(String name) throws InterruptedException {
+ createTopic(name, 1, true);
+ }
+
+ public void waitUntilMetadataIsPropagated(String topic, int numPartitions) {
+ List<KafkaServer> servers = new ArrayList<>();
+ servers.add(kafkaServer);
+ for(int part = 0;part < numPartitions;++part) {
+ TestUtils.waitUntilMetadataIsPropagated(scala.collection.JavaConversions.asScalaBuffer(servers), topic, part, 5000);
+ }
+ }
+
+ public void createTopic(String name, int numPartitions, boolean waitUntilMetadataIsPropagated) throws InterruptedException {
+ AdminUtils.createTopic(zkClient, name, numPartitions, 1, new Properties());
+ if(waitUntilMetadataIsPropagated) {
+ waitUntilMetadataIsPropagated(name, numPartitions);
+ }
+ }
+
+ public void writeMessages(String topic, List<byte[]> messages) {
+ KafkaProducer<String, byte[]> kafkaProducer = createProducer();
+ for(byte[] message: messages) {
+ kafkaProducer.send(new ProducerRecord<String, byte[]>(topic, message));
+ }
+ kafkaProducer.close();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/util/KafkaUtil.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/util/KafkaUtil.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/util/KafkaUtil.java
new file mode 100644
index 0000000..bf2ef4f
--- /dev/null
+++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/util/KafkaUtil.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration.util.integration.util;
+
+
+import kafka.consumer.ConsumerIterator;
+import kafka.consumer.KafkaStream;
+import kafka.javaapi.producer.Producer;
+import kafka.producer.KeyedMessage;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class KafkaUtil {
+ public static <K,V> void send(Producer<K,V> producer, K key, V value, String topic) {
+ producer.send(new KeyedMessage<>(topic, key,value));
+ }
+
+ public static <K,V> void send(Producer<K,V> producer, Iterable<Map.Entry<K,V>> messages, String topic) {
+ for(Map.Entry<K,V> kv : messages) {
+ send(producer, kv.getKey(), kv.getValue(), topic);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/utils/KafkaLoader.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/utils/KafkaLoader.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/utils/KafkaLoader.java
new file mode 100644
index 0000000..4f53e5a
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/utils/KafkaLoader.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.utils;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.storm.flux.Flux;
+import storm.kafka.SpoutConfig;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.util.HashMap;
+import java.util.Map;
+
+public class KafkaLoader {
+
+ private String brokerUrl;
+ private String topic;
+ private String samplePath;
+ private int delay = 1000;
+ private int iterations = -1;
+ private KafkaProducer kafkaProducer;
+
+ public KafkaLoader(String brokerUrl, String topic, String samplePath) {
+ this.brokerUrl = brokerUrl;
+ this.topic = topic;
+ this.samplePath = samplePath;
+ }
+
+ public KafkaLoader withDelay(int delay) {
+ this.delay = delay;
+ return this;
+ }
+
+ public KafkaLoader withIterations(int iterations) {
+ this.iterations = iterations;
+ return this;
+ }
+
+ public void start() {
+ Map<String, Object> producerConfig = new HashMap<>();
+ producerConfig.put("bootstrap.servers", brokerUrl);
+ producerConfig.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ producerConfig.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ kafkaProducer = new KafkaProducer<>(producerConfig);
+ try {
+ while (iterations == -1 || iterations-- > 0) {
+ BufferedReader reader = new BufferedReader(new FileReader(samplePath));
+ String line;
+ while((line = reader.readLine()) != null) {
+ kafkaProducer.send(new ProducerRecord<String, String>(topic, line));
+ Thread.sleep(delay);
+ }
+ reader.close();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void stop() {
+ kafkaProducer.close();
+ }
+
+
+ public static void main(String[] args) {
+ KafkaLoader kafkaLoader = new KafkaLoader(args[0], args[1], args[2]);
+ if (args.length > 3) kafkaLoader.withDelay(Integer.parseInt(args[3]));
+ if (args.length > 4) kafkaLoader.withIterations(Integer.parseInt(args[4]));
+ kafkaLoader.start();
+ kafkaLoader.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/utils/SourceConfigUtils.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/utils/SourceConfigUtils.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/utils/SourceConfigUtils.java
new file mode 100644
index 0000000..ef8b2e2
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/utils/SourceConfigUtils.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.utils;
+
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.metron.Constants;
+import org.apache.metron.domain.SourceConfig;
+import org.apache.zookeeper.KeeperException;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+
+public class SourceConfigUtils {
+
+ public static CuratorFramework getClient(String zookeeperUrl) {
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ return CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy);
+ }
+
+ public static void writeToZookeeperFromFile(String sourceName, String filePath, String zookeeperUrl) throws Exception {
+ writeToZookeeper(sourceName, Files.readAllBytes(Paths.get(filePath)), zookeeperUrl);
+ }
+
+ public static void writeToZookeeper(String sourceName, byte[] configData, String zookeeperUrl) throws Exception {
+ CuratorFramework client = getClient(zookeeperUrl);
+ client.start();
+ try {
+ client.setData().forPath(Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + sourceName, configData);
+ } catch(KeeperException.NoNodeException e) {
+ client.create().creatingParentsIfNeeded().forPath(Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + sourceName, configData);
+ }
+ client.close();
+ }
+
+ public static byte[] readConfigBytesFromZookeeper(String sourceName, String zookeeperUrl) throws Exception {
+ CuratorFramework client = getClient(zookeeperUrl);
+ client.start();
+ byte[] data = client.getData().forPath(Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + sourceName);
+ client.close();
+ return data;
+ }
+
+ public static SourceConfig readConfigFromZookeeper(String sourceName, String zookeeperUrl) throws Exception {
+ byte[] data = readConfigBytesFromZookeeper(sourceName, zookeeperUrl);
+ return SourceConfig.load(new ByteArrayInputStream(data));
+ }
+
+ public static void dumpConfigs(String zookeeperUrl) throws Exception {
+ CuratorFramework client = getClient(zookeeperUrl);
+ client.start();
+ List<String> children = client.getChildren().forPath(Constants.ZOOKEEPER_TOPOLOGY_ROOT);
+ for(String child: children) {
+ byte[] data = client.getData().forPath(Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + child);
+ System.out.println("Config for source " + child);
+ System.out.println(new String(data));
+ System.out.println();
+ }
+ client.close();
+ }
+
+ public static void main(String[] args) {
+ try {
+ File root = new File("./metron-streaming/Metron-Common/src/test/resources/config/source/");
+ for(File child: root.listFiles()) {
+ writeToZookeeperFromFile(child.getName().replaceFirst("-config.json", ""), child.getPath(), "node1:2181");
+ }
+ SourceConfigUtils.dumpConfigs("node1:2181");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/local.yaml
deleted file mode 100644
index 7473b01..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/local.yaml
+++ /dev/null
@@ -1,401 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "asa-local"
-config:
- topology.workers: 1
-
-components:
- - id: "parser"
- className: "org.apache.metron.parsing.parsers.GrokAsaParser"
- - id: "jdbcConfig"
- className: "org.apache.metron.enrichment.adapters.jdbc.MySqlConfig"
- properties:
- - name: "host"
- value: "${mysql.ip}"
- - name: "port"
- value: ${mysql.port}
- - name: "username"
- value: "${mysql.username}"
- - name: "password"
- value: "${mysql.password}"
- - name: "table"
- value: "GEO"
- - id: "geoEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
- configMethods:
- - name: "withJdbcConfig"
- args:
- - ref: "jdbcConfig"
- - id: "geoEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "geo"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "geoEnrichmentAdapter"
- - id: "hostEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
- constructorArgs:
- - '${org.apache.metron.enrichment.host.known_hosts}'
- - id: "hostEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "host"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "hostEnrichmentAdapter"
- - id: "enrichments"
- className: "java.util.ArrayList"
- configMethods:
- - name: "add"
- args:
- - ref: "geoEnrichment"
- - name: "add"
- args:
- - ref: "hostEnrichment"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "alertsConfig"
- className: "java.util.HashMap"
- configMethods:
- - name: "put"
- args: ["whitelist_table_name", "ip_whitelist"]
- - name: "put"
- args: ["blacklist_table_name", "ip_blacklist"]
- - name: "put"
- args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"]
- - name: "put"
- args: ["port", "2181"]
- - name: "put"
- args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"]
- - name: "put"
- args: ["_MAX_TIME_RETAIN_MINUTES", "1000"]
- - id: "alertsAdapter"
- className: "org.apache.metron.alerts.adapters.CIFAlertsAdapter"
- constructorArgs:
- - ref: "alertsConfig"
- - id: "alertsIdentifier"
- className: "org.json.simple.JSONObject"
- configMethods:
- - name: "put"
- args: ["environment", "local"]
- - name: "put"
- args: ["topology", "asa"]
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
-
-spouts:
- - id: "testingSpout"
- className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
- parallelism: 1
- configMethods:
- - name: "withFilename"
- args:
- - "SampleInput/AsaOutput"
- - name: "withRepeating"
- args:
- - true
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "parser"
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "asa_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "asa_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsBolt"
- className: "org.apache.metron.alerts.TelemetryAlertsBolt"
- configMethods:
- - name: "withIdentifier"
- args:
- - ref: "alertsIdentifier"
- - name: "withMaxCacheSize"
- args: [1000]
- - name: "withMaxTimeRetain"
- args: [3600]
- - name: "withAlertsAdapter"
- args:
- - ref: "alertsAdapter"
- - name: "withOutputFieldName"
- args: ["message"]
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "alert"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.ww"
- - name: "withDocumentName"
- args:
- - "asa_alert"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "asa_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "geoEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "geoEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "hostEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "hostEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "joinBolt"
- className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt"
- configMethods:
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
-
-streams:
- - name: "spout -> parser"
- from: "testingSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
- - name: "parser -> host"
- from: "parserBolt"
- to: "hostEnrichmentBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "parser -> geo"
- from: "parserBolt"
- to: "geoEnrichmentBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "parser -> join"
- from: "parserBolt"
- to: "joinBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "geo -> join"
- from: "geoEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "host -> join"
- from: "hostEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "join -> alerts"
- from: "joinBolt"
- to: "alertsBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "alerts -> alertsIndexing"
- from: "alertsBolt"
- to: "alertsIndexingBolt"
- grouping:
- streamId: "message"
- type: SHUFFLE
- - name: "join -> indexing"
- from: "joinBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "alerts -> errors"
- from: "alertsBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/remote.yaml
index 94694ab..78c68d5 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/remote.yaml
@@ -18,146 +18,14 @@ name: "asa"
config:
topology.workers: 1
+
components:
- id: "parser"
className: "org.apache.metron.parsing.parsers.GrokAsaParser"
- - id: "jdbcConfig"
- className: "org.apache.metron.enrichment.adapters.jdbc.MySqlConfig"
- properties:
- - name: "host"
- value: "${mysql.ip}"
- - name: "port"
- value: ${mysql.port}
- - name: "username"
- value: "${mysql.username}"
- - name: "password"
- value: "${mysql.password}"
- - name: "table"
- value: "GEO"
- - id: "geoEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
- configMethods:
- - name: "withJdbcConfig"
- args:
- - ref: "jdbcConfig"
- - id: "geoEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "geo"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "geoEnrichmentAdapter"
- - id: "hostEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
- constructorArgs:
- - '${org.apache.metron.enrichment.host.known_hosts}'
- - id: "hostEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "host"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "hostEnrichmentAdapter"
- - id: "enrichments"
- className: "java.util.ArrayList"
- configMethods:
- - name: "add"
- args:
- - ref: "geoEnrichment"
- - name: "add"
- args:
- - ref: "hostEnrichment"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "alertsConfig"
- className: "java.util.HashMap"
- configMethods:
- - name: "put"
- args: ["whitelist_table_name", "ip_whitelist"]
- - name: "put"
- args: ["blacklist_table_name", "ip_blacklist"]
- - name: "put"
- args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"]
- - name: "put"
- args: ["port", "2181"]
- - name: "put"
- args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"]
- - name: "put"
- args: ["_MAX_TIME_RETAIN_MINUTES", "1000"]
- - id: "alertsAdapter"
- className: "org.apache.metron.alerts.adapters.CIFAlertsAdapter"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
constructorArgs:
- - ref: "alertsConfig"
- - id: "alertsIdentifier"
- className: "org.json.simple.JSONObject"
- configMethods:
- - name: "put"
- args: ["environment", "local"]
- - name: "put"
- args: ["topology", "asa"]
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+ - "${kafka.broker}"
- id: "zkHosts"
className: "storm.kafka.ZkHosts"
constructorArgs:
@@ -168,18 +36,30 @@ components:
# zookeeper hosts
- ref: "zkHosts"
# topic name
- - "${spout.kafka.topic.pcap}"
+ - "${spout.kafka.topic.asa}"
# zk root
- ""
# id
- - "${spout.kafka.topic.pcap}"
+ - "${spout.kafka.topic.asa}"
properties:
- - name: "forceFromStart"
+ - name: "ignoreZkOffsets"
value: true
- name: "startOffsetTime"
value: -1
+ - name: "socketTimeoutMs"
+ value: 1000000
spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - true
- id: "kafkaSpout"
className: "storm.kafka.KafkaSpout"
constructorArgs:
@@ -187,229 +67,16 @@ spouts:
bolts:
- id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "parser"
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "asa_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "asa_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsBolt"
- className: "org.apache.metron.alerts.TelemetryAlertsBolt"
- configMethods:
- - name: "withIdentifier"
- args:
- - ref: "alertsIdentifier"
- - name: "withMaxCacheSize"
- args: [1000]
- - name: "withMaxTimeRetain"
- args: [3600]
- - name: "withAlertsAdapter"
- args:
- - ref: "alertsAdapter"
- - name: "withOutputFieldName"
- args: ["message"]
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "alert"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.ww"
- - name: "withDocumentName"
- args:
- - "asa_alert"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "asa_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "geoEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "geoEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "hostEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "hostEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "joinBolt"
- className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt"
- configMethods:
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "yaf"
+ - ref: "parser"
+ - ref: "writer"
streams:
- - name: "spout -> parser"
+ - name: "spout -> bolt"
from: "kafkaSpout"
to: "parserBolt"
grouping:
type: SHUFFLE
- - name: "parser -> host"
- from: "parserBolt"
- to: "hostEnrichmentBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "parser -> geo"
- from: "parserBolt"
- to: "geoEnrichmentBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "parser -> join"
- from: "parserBolt"
- to: "joinBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "geo -> join"
- from: "geoEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "host -> join"
- from: "hostEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "join -> alerts"
- from: "joinBolt"
- to: "alertsBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "alerts -> alertsIndexing"
- from: "alertsBolt"
- to: "alertsIndexingBolt"
- grouping:
- streamId: "message"
- type: SHUFFLE
- - name: "join -> indexing"
- from: "joinBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "alerts -> errors"
- from: "alertsBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/test.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/test.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/test.yaml
new file mode 100644
index 0000000..9114d94
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/test.yaml
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "asa-test"
+config:
+ topology.workers: 1
+
+
+components:
+ - id: "parser"
+ className: "org.apache.metron.parsing.parsers.GrokAsaParser"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
+ constructorArgs:
+ - "${kafka.broker}"
+ - id: "zkHosts"
+ className: "storm.kafka.ZkHosts"
+ constructorArgs:
+ - "${kafka.zk}"
+ - id: "kafkaConfig"
+ className: "storm.kafka.SpoutConfig"
+ constructorArgs:
+ # zookeeper hosts
+ - ref: "zkHosts"
+ # topic name
+ - "${spout.kafka.topic.asa}"
+ # zk root
+ - ""
+ # id
+ - "${spout.kafka.topic.asa}"
+ properties:
+ - name: "ignoreZkOffsets"
+ value: true
+ - name: "startOffsetTime"
+ value: -2
+ - name: "socketTimeoutMs"
+ value: 1000000
+
+spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - false
+ - id: "kafkaSpout"
+ className: "storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+
+bolts:
+ - id: "parserBolt"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "yaf"
+ - ref: "parser"
+ - ref: "writer"
+
+streams:
+ - name: "spout -> bolt"
+ from: "kafkaSpout"
+ to: "parserBolt"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/local.yaml
deleted file mode 100644
index 851f9d9..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/local.yaml
+++ /dev/null
@@ -1,192 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "bro-local"
-config:
- topology.workers: 1
-
-components:
- - id: "broParser"
- className: "org.apache.metron.parsing.parsers.BasicBroParser"
- - id: "genericMessageFilter"
- className: "org.apache.metron.filters.GenericMessageFilter"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
-
-spouts:
- - id: "testingSpout"
- className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
- parallelism: 1
- configMethods:
- - name: "withFilename"
- args:
- - "SampleInput/BroExampleOutput"
- - name: "withRepeating"
- args:
- - true
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "broParser"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "bro_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "bro_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "bro_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
-
-streams:
- - name: "spout -> parser"
- from: "testingSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
- - name: "parser -> indexing"
- from: "parserBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/remote.yaml
index 96d836e..fb594b5 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/remote.yaml
@@ -18,72 +18,14 @@ name: "bro"
config:
topology.workers: 1
+
components:
- - id: "broParser"
+ - id: "parser"
className: "org.apache.metron.parsing.parsers.BasicBroParser"
- - id: "genericMessageFilter"
- className: "org.apache.metron.filters.GenericMessageFilter"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
+ constructorArgs:
+ - "${kafka.broker}"
- id: "zkHosts"
className: "storm.kafka.ZkHosts"
constructorArgs:
@@ -100,12 +42,24 @@ components:
# id
- "${spout.kafka.topic.bro}"
properties:
- - name: "forceFromStart"
+ - name: "ignoreZkOffsets"
value: true
- name: "startOffsetTime"
value: -1
+ - name: "socketTimeoutMs"
+ value: 1000000
spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - true
- id: "kafkaSpout"
className: "storm.kafka.KafkaSpout"
constructorArgs:
@@ -113,94 +67,16 @@ spouts:
bolts:
- id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "broParser"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "bro_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "bro_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "bro_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "yaf"
+ - ref: "parser"
+ - ref: "writer"
streams:
- - name: "spout -> parser"
+ - name: "spout -> bolt"
from: "kafkaSpout"
to: "parserBolt"
grouping:
type: SHUFFLE
- - name: "parser -> indexing"
- from: "parserBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/test.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/test.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/test.yaml
new file mode 100644
index 0000000..3bd3eed
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/test.yaml
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "bro-test"
+config:
+ topology.workers: 1
+
+
+components:
+ - id: "parser"
+ className: "org.apache.metron.parsing.parsers.BasicBroParser"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
+ constructorArgs:
+ - "${kafka.broker}"
+ - id: "zkHosts"
+ className: "storm.kafka.ZkHosts"
+ constructorArgs:
+ - "${kafka.zk}"
+ - id: "kafkaConfig"
+ className: "storm.kafka.SpoutConfig"
+ constructorArgs:
+ # zookeeper hosts
+ - ref: "zkHosts"
+ # topic name
+ - "${spout.kafka.topic.bro}"
+ # zk root
+ - ""
+ # id
+ - "${spout.kafka.topic.bro}"
+ properties:
+ - name: "ignoreZkOffsets"
+ value: true
+ - name: "startOffsetTime"
+ value: -2
+ - name: "socketTimeoutMs"
+ value: 1000000
+
+spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - false
+ - id: "kafkaSpout"
+ className: "storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+
+bolts:
+ - id: "parserBolt"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "yaf"
+ - ref: "parser"
+ - ref: "writer"
+
+streams:
+ - name: "spout -> bolt"
+ from: "kafkaSpout"
+ to: "parserBolt"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/enrichment/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/enrichment/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/enrichment/remote.yaml
new file mode 100644
index 0000000..8033374
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/enrichment/remote.yaml
@@ -0,0 +1,331 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "enrichment"
+config:
+ topology.workers: 1
+
+components:
+# Enrichment
+ - id: "jdbcConfig"
+ className: "org.apache.metron.enrichment.adapters.jdbc.MySqlConfig"
+ properties:
+ - name: "host"
+ value: "${mysql.ip}"
+ - name: "port"
+ value: ${mysql.port}
+ - name: "username"
+ value: "${mysql.username}"
+ - name: "password"
+ value: "${mysql.password}"
+ - name: "table"
+ value: "GEO"
+ - id: "geoEnrichmentAdapter"
+ className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
+ configMethods:
+ - name: "withJdbcConfig"
+ args:
+ - ref: "jdbcConfig"
+ - id: "geoEnrichment"
+ className: "org.apache.metron.domain.Enrichment"
+ constructorArgs:
+ - "geo"
+ - ref: "geoEnrichmentAdapter"
+ - id: "hostEnrichmentAdapter"
+ className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
+ constructorArgs:
+ - '${org.apache.metron.enrichment.host.known_hosts}'
+ - id: "hostEnrichment"
+ className: "org.apache.metron.domain.Enrichment"
+ constructorArgs:
+ - "host"
+ - ref: "hostEnrichmentAdapter"
+ - id: "enrichments"
+ className: "java.util.ArrayList"
+ configMethods:
+ - name: "add"
+ args:
+ - ref: "geoEnrichment"
+ - name: "add"
+ args:
+ - ref: "hostEnrichment"
+
+# Threat Intel
+ - id: "ipThreatIntelConfig"
+ className: "org.apache.metron.threatintel.ThreatIntelConfig"
+ configMethods:
+ - name: "withProviderImpl"
+ args:
+ - "${hbase.provider.impl}"
+ - name: "withTrackerHBaseTable"
+ args:
+ - "${threat.intel.tracker.table}"
+ - name: "withTrackerHBaseCF"
+ args:
+ - "${threat.intel.tracker.cf}"
+ - name: "withHBaseTable"
+ args:
+ - "${threat.intel.ip.table}"
+ - name: "withHBaseCF"
+ args:
+ - "${threat.intel.ip.cf}"
+ - id: "ipThreatIntelAdapter"
+ className: "org.apache.metron.threatintel.ThreatIntelAdapter"
+ configMethods:
+ - name: "withConfig"
+ args:
+ - ref: "ipThreatIntelConfig"
+ - id: "ipThreatIntelEnrichment"
+ className: "org.apache.metron.domain.Enrichment"
+ constructorArgs:
+ - "ip"
+ - ref: "ipThreatIntelAdapter"
+ - id: "threatIntels"
+ className: "java.util.ArrayList"
+ configMethods:
+ - name: "add"
+ args:
+ - ref: "ipThreatIntelEnrichment"
+
+#indexing
+ - id: "indexWriter"
+ className: "org.apache.metron.writer.ElasticsearchWriter"
+ constructorArgs:
+ - "${es.clustername}"
+ - "${es.ip}"
+ - ${es.port}
+ - "yyyy.MM.dd.hh"
+
+#kafka/zookeeper
+ - id: "zkHosts"
+ className: "storm.kafka.ZkHosts"
+ constructorArgs:
+ - "${kafka.zk}"
+ - id: "kafkaConfig"
+ className: "storm.kafka.SpoutConfig"
+ constructorArgs:
+ # zookeeper hosts
+ - ref: "zkHosts"
+ # topic name
+ - "enrichments"
+ # zk root
+ - ""
+ # id
+ - "enrichments"
+ properties:
+ - name: "ignoreZkOffsets"
+ value: true
+ - name: "startOffsetTime"
+ value: -1
+
+spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - true
+ - id: "kafkaSpout"
+ className: "storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+bolts:
+# Enrichment Bolts
+ - id: "enrichmentSplitBolt"
+ className: "org.apache.metron.enrichment.bolt.EnrichmentSplitterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichments"
+ args:
+ - ref: "enrichments"
+ - id: "geoEnrichmentBolt"
+ className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichment"
+ args:
+ - ref: "geoEnrichment"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+ - id: "hostEnrichmentBolt"
+ className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichment"
+ args:
+ - ref: "hostEnrichment"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+ - id: "enrichmentJoinBolt"
+ className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichments"
+ args:
+ - ref: "enrichments"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+
+# Threat Intel Bolts
+ - id: "threatIntelSplitBolt"
+ className: "org.apache.metron.enrichment.bolt.ThreatIntelSplitterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichments"
+ args:
+ - ref: "threatIntels"
+ - name: "withMessageFieldName"
+ args: ["message"]
+ - id: "ipThreatIntelBolt"
+ className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichment"
+ args:
+ - ref: "ipThreatIntelEnrichment"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+ - id: "threatIntelJoinBolt"
+ className: "org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withEnrichments"
+ args:
+ - ref: "threatIntels"
+ - name: "withMaxCacheSize"
+ args: [10000]
+ - name: "withMaxTimeRetain"
+ args: [10]
+# Indexing Bolts
+ - id: "indexingBolt"
+ className: "org.apache.metron.bolt.BulkMessageWriterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withBulkMessageWriter"
+ args:
+ - ref: "indexWriter"
+
+
+streams:
+#parser
+ - name: "spout -> enrichmentSplit"
+ from: "kafkaSpout"
+ to: "enrichmentSplitBolt"
+ grouping:
+ type: SHUFFLE
+
+#enrichment
+ - name: "enrichmentSplit -> host"
+ from: "enrichmentSplitBolt"
+ to: "hostEnrichmentBolt"
+ grouping:
+ streamId: "host"
+ type: FIELDS
+ args: ["key"]
+ - name: "enrichmentSplit -> geo"
+ from: "enrichmentSplitBolt"
+ to: "geoEnrichmentBolt"
+ grouping:
+ streamId: "geo"
+ type: FIELDS
+ args: ["key"]
+ - name: "splitter -> join"
+ from: "enrichmentSplitBolt"
+ to: "enrichmentJoinBolt"
+ grouping:
+ streamId: "message"
+ type: FIELDS
+ args: ["key"]
+ - name: "geo -> join"
+ from: "geoEnrichmentBolt"
+ to: "enrichmentJoinBolt"
+ grouping:
+ streamId: "geo"
+ type: FIELDS
+ args: ["key"]
+ - name: "host -> join"
+ from: "hostEnrichmentBolt"
+ to: "enrichmentJoinBolt"
+ grouping:
+ streamId: "host"
+ type: FIELDS
+ args: ["key"]
+
+#threat intel
+ - name: "enrichmentJoin -> threatSplit"
+ from: "enrichmentJoinBolt"
+ to: "threatIntelSplitBolt"
+ grouping:
+ streamId: "message"
+ type: FIELDS
+ args: ["key"]
+
+ - name: "threatSplit -> ip"
+ from: "threatIntelSplitBolt"
+ to: "ipThreatIntelBolt"
+ grouping:
+ streamId: "ip"
+ type: FIELDS
+ args: ["key"]
+
+ - name: "ip -> join"
+ from: "ipThreatIntelBolt"
+ to: "threatIntelJoinBolt"
+ grouping:
+ streamId: "ip"
+ type: FIELDS
+ args: ["key"]
+ - name: "threatIntelSplit -> threatIntelJoin"
+ from: "threatIntelSplitBolt"
+ to: "threatIntelJoinBolt"
+ grouping:
+ streamId: "message"
+ type: FIELDS
+ args: ["key"]
+#indexing
+ - name: "threatIntelJoin -> indexing"
+ from: "threatIntelJoinBolt"
+ to: "indexingBolt"
+ grouping:
+ streamId: "message"
+ type: FIELDS
+ args: ["key"]
+ - name: "indexingBolt -> errorIndexingBolt"
+ from: "indexingBolt"
+ to: "indexingBolt"
+ grouping:
+ streamId: "error"
+ type: SHUFFLE
[2/9] incubator-metron git commit: METRON-56 Create unified
enrichment topology (merrimanr via cestella) closes
apache/incubator-metron#33
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/YafExampleOutput
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/YafExampleOutput b/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/YafExampleOutput
index 92b4b4b..8f3ff44 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/YafExampleOutput
+++ b/metron-streaming/Metron-Topologies/src/main/resources/SampleInput/YafExampleOutput
@@ -1,2691 +1,10 @@
-start-time |end-time |duration|rtt |proto|sip |sp |dip |dp |iflags |uflags |riflags |ruflags |isn |risn |tag|rtag|pkt |oct |rpkt |roct |app |end-reason
-2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| 0| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle
-2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| 0| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle
-2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| 0| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle
-2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| 0| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle
-2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle
-2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle
+2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle
+2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle
+2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle
+2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle
+2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle
+2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle
2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle
2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle
2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle
-2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c53037|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.564|2016-01-28 15:29:48.564| 0.000| 0.000| 17| 10.0.2.15|41164| 10.0.2.3| 53| 0| 0| 0| 0|00000000|00000000|000|000| 1| 60| 0| 0| 0|idle
-2016-01-28 15:29:48.575|2016-01-28 15:29:48.575| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|41164| 0| 0| 0| 0|00000000|00000000|000|000| 1| 316| 0| 0| 0|idle
-2016-01-28 15:29:48.575|2016-01-28 15:29:48.575| 0.000| 0.000| 17| 10.0.2.15|37133| 10.0.2.3| 53| 0| 0| 0| 0|00000000|00000000|000|000| 1| 60| 0| 0| 0|idle
-2016-01-28 15:29:48.576|2016-01-28 15:29:48.576| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37133| 0| 0| 0| 0|00000000|00000000|000|000| 1| 88| 0| 0| 0|idle
-2016-01-28 15:29:48.576|2016-01-28 15:29:48.576| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c53037|00000000|000|000| 1| 152| 0| 0| 0|idle
-2016-01-28 15:29:48.577|2016-01-28 15:29:48.577| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa236|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.639|2016-01-28 15:29:48.639| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa236|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.639|2016-01-28 15:29:48.639| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa7c2|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.639|2016-01-28 15:29:48.639| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.639|2016-01-28 15:29:48.639| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efad4e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.639|2016-01-28 15:29:48.639| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efb2da|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.639|2016-01-28 15:29:48.639| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.639|2016-01-28 15:29:48.639| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efb866|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.639|2016-01-28 15:29:48.639| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efbdf2|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.639|2016-01-28 15:29:48.639| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.639|2016-01-28 15:29:48.639| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efc37e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.639|2016-01-28 15:29:48.639| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efc90a|00000000|000|000| 1| 236| 0| 0| 0|idle
-2016-01-28 15:29:48.639|2016-01-28 15:29:48.639| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.640|2016-01-28 15:29:48.640| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efc9ce|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.640|2016-01-28 15:29:48.640| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efcf5a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.640|2016-01-28 15:29:48.640| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.640|2016-01-28 15:29:48.640| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efd4e6|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.640|2016-01-28 15:29:48.640| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efda72|00000000|000|000| 1| 124| 0| 0| 0|idle
-2016-01-28 15:29:48.640|2016-01-28 15:29:48.640| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.642|2016-01-28 15:29:48.642| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efdac6|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.642|2016-01-28 15:29:48.642| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efe052|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.642|2016-01-28 15:29:48.642| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.642|2016-01-28 15:29:48.642| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efe5de|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.642|2016-01-28 15:29:48.642| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efeb6a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.642|2016-01-28 15:29:48.642| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.642|2016-01-28 15:29:48.642| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22eff0f6|00000000|000|000| 1| 152| 0| 0| 0|idle
-2016-01-28 15:29:48.643|2016-01-28 15:29:48.643| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22eff166|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.643|2016-01-28 15:29:48.643| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.643|2016-01-28 15:29:48.643| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22eff6f2|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.643|2016-01-28 15:29:48.643| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22effc7e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.643|2016-01-28 15:29:48.643| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.644|2016-01-28 15:29:48.644| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0020a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.644|2016-01-28 15:29:48.644| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f00796|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.644|2016-01-28 15:29:48.644| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f00d22|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.644|2016-01-28 15:29:48.644| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f012ae|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.644|2016-01-28 15:29:48.644| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0183a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.644|2016-01-28 15:29:48.644| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f01dc6|00000000|000|000| 1| 264| 0| 0| 0|idle
-2016-01-28 15:29:48.644|2016-01-28 15:29:48.644| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f01ea6|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.644|2016-01-28 15:29:48.644| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f02432|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.644|2016-01-28 15:29:48.644| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f029be|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.644|2016-01-28 15:29:48.644| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f02f4a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.644|2016-01-28 15:29:48.644| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f034d6|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.644|2016-01-28 15:29:48.644| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f03a62|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.644|2016-01-28 15:29:48.644| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f03fee|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.646|2016-01-28 15:29:48.646| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0457a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f04b06|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f05092|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0561e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f05baa|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f06136|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f066c2|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f06c4e|00000000|000|000| 1| 432| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f06dd6|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f07362|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f078ee|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f07e7a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f08406|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f08992|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f08f1e|00000000|000|000| 1| 208| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f08fc6|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f09552|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f09ade|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0a06a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0a5f6|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0ab82|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0b10e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0b69a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0bc26|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0c1b2|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0c73e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0ccca|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0d256|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0d7e2|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0dd6e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0e2fa|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0e886|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0ee12|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0f39e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.647|2016-01-28 15:29:48.647| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.648|2016-01-28 15:29:48.648| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0f92a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.648|2016-01-28 15:29:48.648| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f0feb6|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.648|2016-01-28 15:29:48.648| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f10442|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.648|2016-01-28 15:29:48.648| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f109ce|00000000|000|000| 1| 656| 0| 0| 0|idle
-2016-01-28 15:29:48.648|2016-01-28 15:29:48.648| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.648|2016-01-28 15:29:48.648| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f10c36|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.648|2016-01-28 15:29:48.648| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f111c2|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.648|2016-01-28 15:29:48.648| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.648|2016-01-28 15:29:48.648| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1174e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.648|2016-01-28 15:29:48.648| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f11cda|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.649|2016-01-28 15:29:48.649| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f12266|00000000|000|000| 1| 152| 0| 0| 0|idle
-2016-01-28 15:29:48.649|2016-01-28 15:29:48.649| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f122d6|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.649|2016-01-28 15:29:48.649| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f12862|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.649|2016-01-28 15:29:48.649| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f12dee|00000000|000|000| 1| 96| 0| 0| 0|idle
-2016-01-28 15:29:48.649|2016-01-28 15:29:48.649| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f12e26|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f133b2|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1393e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f13eca|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f14456|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f149e2|00000000|000|000| 1| 180| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f14a6e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f14ffa|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f15586|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f15b12|00000000|000|000| 1| 124| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f15b66|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f160f2|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f1667e|00000000|000|000| 1| 96| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f166b6|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f16c42|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f171ce|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1775a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f17ce6|00000000|000|000| 1| 152| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f17d56|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f182e2|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1886e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f18dfa|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f19386|00000000|000|000| 1| 152| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f193f6|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f19982|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f19f0e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1a49a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f1aa26|00000000|000|000| 1| 152| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.651|2016-01-28 15:29:48.651| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1aa96|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1b022|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1b5ae|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1bb3a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1c0c6|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1c652|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f1cbde|00000000|000|000| 1| 208| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1cc86|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1d212|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f1d79e|00000000|000|000| 1| 96| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1d7d6|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1dd62|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1e2ee|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1e87a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f1ee06|00000000|000|000| 1| 152| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1ee76|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1f402|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f1f98e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22f1ff1a|00000000|000|000| 1| 642| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AF| 0| 0| 0|22f20174|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AF| 0| 0| 0|58c530a7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.653|2016-01-28 15:29:48.653| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22f20175|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.733|2016-01-28 15:29:48.733| 0.000| 0.000| 17| 10.0.2.15|43106| 10.0.2.3| 53| 0| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle
-2016-01-28 15:29:48.735|2016-01-28 15:29:48.735| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|43106| 0| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle
-2016-01-28 15:29:48.735|2016-01-28 15:29:48.735| 0.000| 0.000| 17| 10.0.2.15|37775| 10.0.2.3| 53| 0| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle
-2016-01-28 15:29:48.736|2016-01-28 15:29:48.736| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37775| 0| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle
-2016-01-28 15:29:48.737|2016-01-28 15:29:48.737| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| S| 0| 0| 0|d9632bff|00000000|000|000| 1| 60| 0| 0| 0|idle
-2016-01-28 15:29:48.741|2016-01-28 15:29:48.741| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AS| 0| 0| 0|22f19401|00000000|000|000| 1| 44| 0| 0| 0|idle
-2016-01-28 15:29:48.741|2016-01-28 15:29:48.741| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632c00|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.741|2016-01-28 15:29:48.741| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| AP| 0| 0| 0|d9632c00|00000000|000|000| 1| 148| 0| 0| 0|idle
-2016-01-28 15:29:48.741|2016-01-28 15:29:48.741| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f19402|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.790|2016-01-28 15:29:48.790| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f19402|00000000|000|000| 1| 604| 0| 0| 0|idle
-2016-01-28 15:29:48.790|2016-01-28 15:29:48.790| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632c6c|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.792|2016-01-28 15:29:48.792| 0.000| 0.000| 17| 10.0.2.15|59684| 10.0.2.3| 53| 0| 0| 0| 0|00000000|00000000|000|000| 1| 60| 0| 0| 0|idle
-2016-01-28 15:29:48.793|2016-01-28 15:29:48.793| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|59684| 0| 0| 0| 0|00000000|00000000|000|000| 1| 316| 0| 0| 0|idle
-2016-01-28 15:29:48.793|2016-01-28 15:29:48.793| 0.000| 0.000| 17| 10.0.2.15|59198| 10.0.2.3| 53| 0| 0| 0| 0|00000000|00000000|000|000| 1| 60| 0| 0| 0|idle
-2016-01-28 15:29:48.794|2016-01-28 15:29:48.794| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|59198| 0| 0| 0| 0|00000000|00000000|000|000| 1| 88| 0| 0| 0|idle
-2016-01-28 15:29:48.795|2016-01-28 15:29:48.795| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| AP| 0| 0| 0|d9632c6c|00000000|000|000| 1| 152| 0| 0| 0|idle
-2016-01-28 15:29:48.795|2016-01-28 15:29:48.795| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f19636|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.859|2016-01-28 15:29:48.859| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f19636|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.859|2016-01-28 15:29:48.859| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f19bc2|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.859|2016-01-28 15:29:48.859| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.859|2016-01-28 15:29:48.859| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1a14e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.860|2016-01-28 15:29:48.860| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1a6da|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.860|2016-01-28 15:29:48.860| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.860|2016-01-28 15:29:48.860| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1ac66|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.860|2016-01-28 15:29:48.860| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1b1f2|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.860|2016-01-28 15:29:48.860| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.860|2016-01-28 15:29:48.860| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1b77e|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.860|2016-01-28 15:29:48.860| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1bd0a|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.860|2016-01-28 15:29:48.860| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.860|2016-01-28 15:29:48.860| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1c296|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.860|2016-01-28 15:29:48.860| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f1c822|00000000|000|000| 1| 709| 0| 0| 0|idle
-2016-01-28 15:29:48.860|2016-01-28 15:29:48.860| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.863|2016-01-28 15:29:48.863| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1cabf|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.863|2016-01-28 15:29:48.863| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1d04b|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.863|2016-01-28 15:29:48.863| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.863|2016-01-28 15:29:48.863| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1d5d7|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.863|2016-01-28 15:29:48.863| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1db63|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.863|2016-01-28 15:29:48.863| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.863|2016-01-28 15:29:48.863| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f1e0ef|00000000|000|000| 1| 152| 0| 0| 0|idle
-2016-01-28 15:29:48.863|2016-01-28 15:29:48.863| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1e15f|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.863|2016-01-28 15:29:48.863| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.863|2016-01-28 15:29:48.863| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f1e6eb|00000000|000|000| 1| 68| 0| 0| 0|idle
-2016-01-28 15:29:48.864|2016-01-28 15:29:48.864| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1e707|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.864|2016-01-28 15:29:48.864| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1ec93|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1f21f|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1f7ab|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f1fd37|00000000|000|000| 1| 152| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f1fda7|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f20333|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f208bf|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f20e4b|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f213d7|00000000|000|000| 1| 152| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f21447|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f219d3|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f21f5f|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f224eb|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f22a77|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f23003|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2358f|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f23b1b|00000000|000|000| 1| 236| 0| 0| 0|idle
-2016-01-28 15:29:48.865|2016-01-28 15:29:48.865| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.866|2016-01-28 15:29:48.866| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f23bdf|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.866|2016-01-28 15:29:48.866| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2416b|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.866|2016-01-28 15:29:48.866| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.866|2016-01-28 15:29:48.866| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f246f7|00000000|000|000| 1| 96| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2472f|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f24cbb|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f25247|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f257d3|00000000|000|000| 1| 124| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f25827|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f25db3|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2633f|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f268cb|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f26e57|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f273e3|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2796f|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f27efb|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f28487|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f28a13|00000000|000|000| 1| 292| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f28b0f|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2909b|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f29627|00000000|000|000| 1| 96| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2965f|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f29beb|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f2a177|00000000|000|000| 1| 96| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2a1af|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2a73b|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2acc7|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2b253|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2b7df|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2bd6b|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2c2f7|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2c883|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.869|2016-01-28 15:29:48.869| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2ce0f|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2d39b|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2d927|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2deb3|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2e43f|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2e9cb|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2ef57|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2f4e3|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2fa6f|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f2fffb|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f30587|00000000|000|000| 1| 544| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f3077f|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f30d0b|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f31297|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f31823|00000000|000|000| 1| 124| 0| 0| 0|idle
-2016-01-28 15:29:48.870|2016-01-28 15:29:48.870| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.871|2016-01-28 15:29:48.871| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f31877|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.871|2016-01-28 15:29:48.871| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f31e03|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.871|2016-01-28 15:29:48.871| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.871|2016-01-28 15:29:48.871| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f3238f|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.871|2016-01-28 15:29:48.871| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f3291b|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.871|2016-01-28 15:29:48.871| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f32ea7|00000000|000|000| 1| 152| 0| 0| 0|idle
-2016-01-28 15:29:48.871|2016-01-28 15:29:48.871| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.872|2016-01-28 15:29:48.872| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f32f17|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.872|2016-01-28 15:29:48.872| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f334a3|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.872|2016-01-28 15:29:48.872| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.873|2016-01-28 15:29:48.873| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f33a2f|00000000|000|000| 1| 96| 0| 0| 0|idle
-2016-01-28 15:29:48.873|2016-01-28 15:29:48.873| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f33a67|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.873|2016-01-28 15:29:48.873| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f33ff3|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.873|2016-01-28 15:29:48.873| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f3457f|00000000|000|000| 1| 96| 0| 0| 0|idle
-2016-01-28 15:29:48.873|2016-01-28 15:29:48.873| 0.000| 0.000| 6| 10.0.2.15|50379| 216.21.170.217| 80| A| 0| 0| 0|d9632cdc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2016-01-28 15:29:48.874|2016-01-28 15:29:48.874| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f345b7|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.874|2016-01-28 15:29:48.874| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f34b43|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.874|2016-01-28 15:29:48.874| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f350cf|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.874|2016-01-28 15:29:48.874| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f3565b|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.874|2016-01-28 15:29:48.874| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f35be7|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.874|2016-01-28 15:29:48.874| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f36173|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.874|2016-01-28 15:29:48.874| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f366ff|00000000|000|000| 1| 208| 0| 0| 0|idle
-2016-01-28 15:29:48.874|2016-01-28 15:29:48.874| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f367a7|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.874|2016-01-28 15:29:48.874| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f36d33|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.874|2016-01-28 15:29:48.874| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| AP| 0| 0| 0|22f372bf|00000000|000|000| 1| 96| 0| 0| 0|idle
-2016-01-28 15:29:48.874|2016-01-28 15:29:48.874| 0.000| 0.000| 6| 216.21.170.217| 80| 10.0.2.15|50379| A| 0| 0| 0|22f372f7|00000000|000|000| 1| 1460| 0| 0| 0|idle
-2016-01-28 15:29:48.874|2016-01
<TRUNCATED>
[4/9] incubator-metron git commit: METRON-56 Create unified
enrichment topology (merrimanr via cestella) closes
apache/incubator-metron#33
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/remote.yaml
index 3f1f57b..6caa7fe 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/remote.yaml
@@ -21,143 +21,10 @@ config:
components:
- id: "parser"
className: "org.apache.metron.parsing.parsers.BasicLancopeParser"
- - id: "jdbcConfig"
- className: "org.apache.metron.enrichment.adapters.jdbc.MySqlConfig"
- properties:
- - name: "host"
- value: "${mysql.ip}"
- - name: "port"
- value: ${mysql.port}
- - name: "username"
- value: "${mysql.username}"
- - name: "password"
- value: "${mysql.password}"
- - name: "table"
- value: "GEO"
- - id: "geoEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
- configMethods:
- - name: "withJdbcConfig"
- args:
- - ref: "jdbcConfig"
- - id: "geoEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "geo"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "geoEnrichmentAdapter"
- - id: "hostEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
constructorArgs:
- - '${org.apache.metron.enrichment.host.known_hosts}'
- - id: "hostEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "host"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "hostEnrichmentAdapter"
- - id: "enrichments"
- className: "java.util.ArrayList"
- configMethods:
- - name: "add"
- args:
- - ref: "geoEnrichment"
- - name: "add"
- args:
- - ref: "hostEnrichment"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "alertsConfig"
- className: "java.util.HashMap"
- configMethods:
- - name: "put"
- args: ["whitelist_table_name", "ip_whitelist"]
- - name: "put"
- args: ["blacklist_table_name", "ip_blacklist"]
- - name: "put"
- args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"]
- - name: "put"
- args: ["port", "2181"]
- - name: "put"
- args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"]
- - name: "put"
- args: ["_MAX_TIME_RETAIN_MINUTES", "1000"]
- - id: "alertsAdapter"
- className: "org.apache.metron.alerts.adapters.CIFAlertsAdapter"
- constructorArgs:
- - ref: "alertsConfig"
- - id: "alertsIdentifier"
- className: "org.json.simple.JSONObject"
- configMethods:
- - name: "put"
- args: ["environment", "local"]
- - name: "put"
- args: ["topology", "lancope"]
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+ - "${kafka.broker}"
- id: "zkHosts"
className: "storm.kafka.ZkHosts"
constructorArgs:
@@ -168,18 +35,28 @@ components:
# zookeeper hosts
- ref: "zkHosts"
# topic name
- - "${spout.kafka.topic.pcap}"
+ - "${spout.kafka.topic.snort}"
# zk root
- ""
# id
- - "${spout.kafka.topic.pcap}"
+ - "${spout.kafka.topic.snort}"
properties:
- - name: "forceFromStart"
+ - name: "ignoreZkOffsets"
value: true
- name: "startOffsetTime"
value: -1
spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - true
- id: "kafkaSpout"
className: "storm.kafka.KafkaSpout"
constructorArgs:
@@ -187,229 +64,16 @@ spouts:
bolts:
- id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "parser"
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "lancope_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "lancope_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsBolt"
- className: "org.apache.metron.alerts.TelemetryAlertsBolt"
- configMethods:
- - name: "withIdentifier"
- args:
- - ref: "alertsIdentifier"
- - name: "withMaxCacheSize"
- args: [1000]
- - name: "withMaxTimeRetain"
- args: [3600]
- - name: "withAlertsAdapter"
- args:
- - ref: "alertsAdapter"
- - name: "withOutputFieldName"
- args: ["message"]
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "alert"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.ww"
- - name: "withDocumentName"
- args:
- - "lancope_alert"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "lancope_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "geoEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "geoEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "hostEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "hostEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "joinBolt"
- className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt"
- configMethods:
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "${spout.kafka.topic.snort}"
+ - ref: "parser"
+ - ref: "writer"
streams:
- - name: "spout -> parser"
+ - name: "spout -> bolt"
from: "kafkaSpout"
to: "parserBolt"
grouping:
type: SHUFFLE
- - name: "parser -> host"
- from: "parserBolt"
- to: "hostEnrichmentBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "parser -> geo"
- from: "parserBolt"
- to: "geoEnrichmentBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "parser -> join"
- from: "parserBolt"
- to: "joinBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "geo -> join"
- from: "geoEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "host -> join"
- from: "hostEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "join -> alerts"
- from: "joinBolt"
- to: "alertsBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "alerts -> alertsIndexing"
- from: "alertsBolt"
- to: "alertsIndexingBolt"
- grouping:
- streamId: "message"
- type: SHUFFLE
- - name: "join -> indexing"
- from: "joinBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "alerts -> errors"
- from: "alertsBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/test.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/test.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/test.yaml
new file mode 100644
index 0000000..e27e02b
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/test.yaml
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "lancope-test"
+config:
+ topology.workers: 1
+
+components:
+ - id: "parser"
+ className: "org.apache.metron.parsing.parsers.BasicLancopeParser"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
+ constructorArgs:
+ - "${kafka.broker}"
+ - id: "zkHosts"
+ className: "storm.kafka.ZkHosts"
+ constructorArgs:
+ - "${kafka.zk}"
+ - id: "kafkaConfig"
+ className: "storm.kafka.SpoutConfig"
+ constructorArgs:
+ # zookeeper hosts
+ - ref: "zkHosts"
+ # topic name
+ - "${spout.kafka.topic.snort}"
+ # zk root
+ - ""
+ # id
+ - "${spout.kafka.topic.snort}"
+ properties:
+ - name: "ignoreZkOffsets"
+ value: true
+ - name: "startOffsetTime"
+ value: -2
+
+spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - false
+ - id: "kafkaSpout"
+ className: "storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+
+bolts:
+ - id: "parserBolt"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "${spout.kafka.topic.snort}"
+ - ref: "parser"
+ - ref: "writer"
+
+streams:
+ - name: "spout -> bolt"
+ from: "kafkaSpout"
+ to: "parserBolt"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/local.yaml
deleted file mode 100644
index 45e8102..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/local.yaml
+++ /dev/null
@@ -1,172 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "paloalto-local"
-config:
- topology.workers: 1
-
-components:
- - id: "paloAltoParser"
- className: "org.apache.metron.parsing.parsers.BasicPaloAltoFirewallParser"
- - id: "genericMessageFilter"
- className: "org.apache.metron.filters.GenericMessageFilter"
- - id: "filenameFormat"
- className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat"
- configMethods:
- - name: "withPath"
- args:
- - "${bolt.hdfs.wip.file.path}"
- - id: "messageField"
- className: "backtype.storm.tuple.Fields"
- constructorArgs:
- - ["message"]
- - id: "recordFormat"
- className: "org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat"
- configMethods:
- - name: "withFieldDelimiter"
- args:
- - "${bolt.hdfs.field.delimiter}"
- - name: "withFields"
- args:
- - ref: "messageField"
- - id: "rotationPolicy"
- className: "org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy"
- constructorArgs:
- - ${bolt.hdfs.file.rotation.size.in.mb}
- - MB
- - id: "syncPolicy"
- className: "org.apache.storm.hdfs.bolt.sync.CountSyncPolicy"
- constructorArgs:
- - ${bolt.hdfs.batch.size}
- - id: "moveFileAction"
- className: "org.apache.storm.hdfs.common.rotation.MoveFileAction"
- configMethods:
- - name: "toDestination"
- args:
- - "${bolt.hdfs.finished.file.path}"
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
-
-spouts:
- - id: "testingSpout"
- className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
- parallelism: 1
- configMethods:
- - name: "withFilename"
- args:
- - "SampleInput/PaloaltoOutput"
- - name: "withRepeating"
- args:
- - true
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "paloAltoParser"
- - id: "hdfsBolt"
- className: "org.apache.storm.hdfs.bolt.HdfsBolt"
- configMethods:
- - name: "withFsUrl"
- args:
- - "${bolt.hdfs.file.system.url}"
- - name: "withFileNameFormat"
- args:
- - ref: "filenameFormat"
- - name: "withRecordFormat"
- args:
- - ref: "recordFormat"
- - name: "withRotationPolicy"
- args:
- - ref: "rotationPolicy"
- - name: "withSyncPolicy"
- args:
- - ref: "syncPolicy"
- - name: "addRotationAction"
- args:
- - ref: "moveFileAction"
-
-streams:
- - name: "spout -> parser"
- from: "testingSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
- - name: "parser -> hdfs"
- from: "parserBolt"
- to: "hdfsBolt"
- grouping:
- streamId: "message"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/remote.yaml
index 4f42084..1e7933c 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/remote.yaml
@@ -19,103 +19,12 @@ config:
topology.workers: 1
components:
- - id: "paloAltoParser"
+ - id: "parser"
className: "org.apache.metron.parsing.parsers.BasicPaloAltoFirewallParser"
- - id: "genericMessageFilter"
- className: "org.apache.metron.filters.GenericMessageFilter"
- - id: "filenameFormat"
- className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat"
- configMethods:
- - name: "withPath"
- args:
- - "${bolt.hdfs.wip.file.path}"
- - id: "messageField"
- className: "backtype.storm.tuple.Fields"
- constructorArgs:
- - ["message"]
- - id: "recordFormat"
- className: "org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat"
- configMethods:
- - name: "withFieldDelimiter"
- args:
- - "${bolt.hdfs.field.delimiter}"
- - name: "withFields"
- args:
- - ref: "messageField"
- - id: "rotationPolicy"
- className: "org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy"
- constructorArgs:
- - ${bolt.hdfs.file.rotation.size.in.mb}
- - MB
- - id: "syncPolicy"
- className: "org.apache.storm.hdfs.bolt.sync.CountSyncPolicy"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
constructorArgs:
- - ${bolt.hdfs.batch.size}
- - id: "moveFileAction"
- className: "org.apache.storm.hdfs.common.rotation.MoveFileAction"
- configMethods:
- - name: "toDestination"
- args:
- - "${bolt.hdfs.finished.file.path}"
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+ - "${kafka.broker}"
- id: "zkHosts"
className: "storm.kafka.ZkHosts"
constructorArgs:
@@ -126,18 +35,28 @@ components:
# zookeeper hosts
- ref: "zkHosts"
# topic name
- - "${spout.kafka.topic.paloalto}"
+ - "${spout.kafka.topic.snort}"
# zk root
- ""
# id
- - "${spout.kafka.topic.paloalto}"
+ - "${spout.kafka.topic.snort}"
properties:
- - name: "forceFromStart"
+ - name: "ignoreZkOffsets"
value: true
- name: "startOffsetTime"
value: -1
spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - true
- id: "kafkaSpout"
className: "storm.kafka.KafkaSpout"
constructorArgs:
@@ -145,42 +64,16 @@ spouts:
bolts:
- id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "paloAltoParser"
- - id: "hdfsBolt"
- className: "org.apache.storm.hdfs.bolt.HdfsBolt"
- configMethods:
- - name: "withFsUrl"
- args:
- - "${bolt.hdfs.file.system.url}"
- - name: "withFileNameFormat"
- args:
- - ref: "filenameFormat"
- - name: "withRecordFormat"
- args:
- - ref: "recordFormat"
- - name: "withRotationPolicy"
- args:
- - ref: "rotationPolicy"
- - name: "withSyncPolicy"
- args:
- - ref: "syncPolicy"
- - name: "addRotationAction"
- args:
- - ref: "moveFileAction"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "${spout.kafka.topic.snort}"
+ - ref: "parser"
+ - ref: "writer"
streams:
- - name: "spout -> parser"
+ - name: "spout -> bolt"
from: "kafkaSpout"
to: "parserBolt"
grouping:
type: SHUFFLE
- - name: "parser -> hdfs"
- from: "parserBolt"
- to: "hdfsBolt"
- grouping:
- streamId: "message"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/test.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/test.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/test.yaml
new file mode 100644
index 0000000..e56e16f
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/test.yaml
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "paloalto-test"
+config:
+ topology.workers: 1
+
+components:
+ - id: "parser"
+ className: "org.apache.metron.parsing.parsers.BasicPaloAltoFirewallParser"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
+ constructorArgs:
+ - "${kafka.broker}"
+ - id: "zkHosts"
+ className: "storm.kafka.ZkHosts"
+ constructorArgs:
+ - "${kafka.zk}"
+ - id: "kafkaConfig"
+ className: "storm.kafka.SpoutConfig"
+ constructorArgs:
+ # zookeeper hosts
+ - ref: "zkHosts"
+ # topic name
+ - "${spout.kafka.topic.snort}"
+ # zk root
+ - ""
+ # id
+ - "${spout.kafka.topic.snort}"
+ properties:
+ - name: "ignoreZkOffsets"
+ value: true
+ - name: "startOffsetTime"
+ value: -2
+
+spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - false
+ - id: "kafkaSpout"
+ className: "storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+
+bolts:
+ - id: "parserBolt"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "${spout.kafka.topic.snort}"
+ - ref: "parser"
+ - ref: "writer"
+
+streams:
+ - name: "spout -> bolt"
+ from: "kafkaSpout"
+ to: "parserBolt"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/local.yaml
index a8848fe..3987a18 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/local.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/local.yaml
@@ -53,7 +53,7 @@ components:
- id: "ipThreatIntelEnrichment"
className: "org.apache.metron.domain.Enrichment"
properties:
- - name: "name"
+ - name: "type"
value: "ip"
- name: "fields"
value: ["message/ip_src_addr", "message/ip_dst_addr"]
@@ -71,7 +71,7 @@ components:
- id: "geoEnrichment"
className: "org.apache.metron.domain.Enrichment"
properties:
- - name: "name"
+ - name: "type"
value: "geo"
- name: "fields"
value: ["ip_src_addr", "ip_dst_addr"]
@@ -84,7 +84,7 @@ components:
- id: "hostEnrichment"
className: "org.apache.metron.domain.Enrichment"
properties:
- - name: "name"
+ - name: "type"
value: "host"
- name: "fields"
value: ["ip_src_addr", "ip_dst_addr"]
@@ -274,7 +274,7 @@ bolts:
- ref: "metricConfig"
# Threat Intel Bolts
- id: "threatIntelSplitBolt"
- className: "org.apache.metron.enrichment.EnrichmentSplitterBolt"
+ className: "org.apache.metron.enrichment.bolt.EnrichmentSplitterBolt"
configMethods:
- name: "withEnrichments"
args:
@@ -342,13 +342,13 @@ streams:
grouping:
type: SHUFFLE
#hbase
- - name: "parser -> hbase"
- from: "parserBolt"
- to: "hbaseBolt"
- grouping:
- streamId: "raw"
- type: FIELDS
- args: ["key"]
+# - name: "parser -> hbase"
+# from: "parserBolt"
+# to: "hbaseBolt"
+# grouping:
+# streamId: "raw"
+# type: FIELDS
+# args: ["key"]
#enrichment
- name: "parser -> host"
from: "parserBolt"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/parse.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/parse.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/parse.yaml
new file mode 100644
index 0000000..dabaa7d
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/parse.yaml
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "yaf-test"
+config:
+ topology.workers: 1
+
+components:
+ - id: "parser"
+ className: "org.apache.metron.parsing.parsers.PcapParser"
+ - id: "writer"
+ className: "org.apache.metron.writer.PcapWriter"
+ constructorArgs:
+ - "${bolt.hbase.table.name}"
+ - "${bolt.hbase.table.fields}"
+ - id: "zkHosts"
+ className: "storm.kafka.ZkHosts"
+ constructorArgs:
+ - "${kafka.zk}"
+ - id: "kafkaConfig"
+ className: "storm.kafka.SpoutConfig"
+ constructorArgs:
+ # zookeeper hosts
+ - ref: "zkHosts"
+ # topic name
+ - "${spout.kafka.topic.pcap}"
+ # zk root
+ - ""
+ # id
+ - "${spout.kafka.topic.pcap}"
+ properties:
+ - name: "ignoreZkOffsets"
+ value: true
+ - name: "startOffsetTime"
+ value: -2
+
+spouts:
+ - id: "kafkaSpout"
+ className: "storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+
+bolts:
+ - id: "parserBolt"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "pcap"
+ - ref: "parser"
+ - ref: "writer"
+
+streams:
+ - name: "spout -> bolt"
+ from: "kafkaSpout"
+ to: "parserBolt"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/remote.yaml
index e170895..f7b0f20 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/remote.yaml
@@ -295,7 +295,7 @@ bolts:
- ref: "metricConfig"
# Threat Intel Bolts
- id: "threatIntelSplitBolt"
- className: "org.apache.metron.enrichment.EnrichmentSplitterBolt"
+ className: "org.apache.metron.enrichment.bolt.EnrichmentSplitterBolt"
configMethods:
- name: "withEnrichments"
args:
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/local.yaml
deleted file mode 100644
index 6281d5b..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/local.yaml
+++ /dev/null
@@ -1,195 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "snort-local"
-config:
- topology.workers: 1
-
-components:
- - id: "snortParser"
- className: "org.apache.metron.parsing.parsers.BasicSnortParser"
- - id: "genericMessageFilter"
- className: "org.apache.metron.filters.GenericMessageFilter"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
-
-
-spouts:
- - id: "testingSpout"
- className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
- parallelism: 1
- configMethods:
- - name: "withFilename"
- args:
- - "SampleInput/SourcefireExampleOutput"
- - name: "withRepeating"
- args:
- - true
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "snortParser"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "snort_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "snort_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "snort_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
-
-streams:
- - name: "spout -> parser"
- from: "testingSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
- - name: "parser -> indexing"
- from: "parserBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
-
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/remote.yaml
index 2bfadd0..7f52d0f 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/remote.yaml
@@ -19,71 +19,12 @@ config:
topology.workers: 1
components:
- - id: "snortParser"
+ - id: "parser"
className: "org.apache.metron.parsing.parsers.BasicSnortParser"
- - id: "genericMessageFilter"
- className: "org.apache.metron.filters.GenericMessageFilter"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
+ constructorArgs:
+ - "${kafka.broker}"
- id: "zkHosts"
className: "storm.kafka.ZkHosts"
constructorArgs:
@@ -100,12 +41,22 @@ components:
# id
- "${spout.kafka.topic.snort}"
properties:
- - name: "forceFromStart"
+ - name: "ignoreZkOffsets"
value: true
- name: "startOffsetTime"
value: -1
spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - true
- id: "kafkaSpout"
className: "storm.kafka.KafkaSpout"
constructorArgs:
@@ -113,96 +64,16 @@ spouts:
bolts:
- id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "snortParser"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "snort_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "snort_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "snort_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "${spout.kafka.topic.snort}"
+ - ref: "parser"
+ - ref: "writer"
streams:
- - name: "spout -> parser"
+ - name: "spout -> bolt"
from: "kafkaSpout"
to: "parserBolt"
grouping:
type: SHUFFLE
- - name: "parser -> indexing"
- from: "parserBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
-
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/test.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/test.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/test.yaml
new file mode 100644
index 0000000..bdbea97
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/test.yaml
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "snort-test"
+config:
+ topology.workers: 1
+
+components:
+ - id: "parser"
+ className: "org.apache.metron.parsing.parsers.BasicSnortParser"
+ - id: "writer"
+ className: "org.apache.metron.writer.KafkaWriter"
+ constructorArgs:
+ - "${kafka.broker}"
+ - id: "zkHosts"
+ className: "storm.kafka.ZkHosts"
+ constructorArgs:
+ - "${kafka.zk}"
+ - id: "kafkaConfig"
+ className: "storm.kafka.SpoutConfig"
+ constructorArgs:
+ # zookeeper hosts
+ - ref: "zkHosts"
+ # topic name
+ - "${spout.kafka.topic.snort}"
+ # zk root
+ - ""
+ # id
+ - "${spout.kafka.topic.snort}"
+ properties:
+ - name: "ignoreZkOffsets"
+ value: true
+ - name: "startOffsetTime"
+ value: -2
+
+spouts:
+ - id: "testingSpout"
+ className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+ parallelism: 1
+ configMethods:
+ - name: "withFilename"
+ args:
+ - "SampleInput/YafExampleOutput"
+ - name: "withRepeating"
+ args:
+ - false
+ - id: "kafkaSpout"
+ className: "storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+
+bolts:
+ - id: "parserBolt"
+ className: "org.apache.metron.bolt.ParserBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ - "${spout.kafka.topic.snort}"
+ - ref: "parser"
+ - ref: "writer"
+
+streams:
+ - name: "spout -> bolt"
+ from: "kafkaSpout"
+ to: "parserBolt"
+ grouping:
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9f96399d/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/local.yaml
deleted file mode 100644
index 6464563..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/sourcefire/local.yaml
+++ /dev/null
@@ -1,401 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "sourcefire-local"
-config:
- topology.workers: 1
-
-components:
- - id: "parser"
- className: "org.apache.metron.parsing.parsers.BasicSourcefireParser"
- - id: "jdbcConfig"
- className: "org.apache.metron.enrichment.adapters.jdbc.MySqlConfig"
- properties:
- - name: "host"
- value: "${mysql.ip}"
- - name: "port"
- value: ${mysql.port}
- - name: "username"
- value: "${mysql.username}"
- - name: "password"
- value: "${mysql.password}"
- - name: "table"
- value: "GEO"
- - id: "geoEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
- configMethods:
- - name: "withJdbcConfig"
- args:
- - ref: "jdbcConfig"
- - id: "geoEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "geo"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "geoEnrichmentAdapter"
- - id: "hostEnrichmentAdapter"
- className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
- constructorArgs:
- - '${org.apache.metron.enrichment.host.known_hosts}'
- - id: "hostEnrichment"
- className: "org.apache.metron.domain.Enrichment"
- properties:
- - name: "name"
- value: "host"
- - name: "fields"
- value: ["ip_src_addr", "ip_dst_addr"]
- - name: "adapter"
- ref: "hostEnrichmentAdapter"
- - id: "enrichments"
- className: "java.util.ArrayList"
- configMethods:
- - name: "add"
- args:
- - ref: "geoEnrichment"
- - name: "add"
- args:
- - ref: "hostEnrichment"
- - id: "indexAdapter"
- className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
- - id: "alertsConfig"
- className: "java.util.HashMap"
- configMethods:
- - name: "put"
- args: ["whitelist_table_name", "ip_whitelist"]
- - name: "put"
- args: ["blacklist_table_name", "ip_blacklist"]
- - name: "put"
- args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"]
- - name: "put"
- args: ["port", "2181"]
- - name: "put"
- args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"]
- - name: "put"
- args: ["_MAX_TIME_RETAIN_MINUTES", "1000"]
- - id: "alertsAdapter"
- className: "org.apache.metron.alerts.adapters.CIFAlertsAdapter"
- constructorArgs:
- - ref: "alertsConfig"
- - id: "alertsIdentifier"
- className: "org.json.simple.JSONObject"
- configMethods:
- - name: "put"
- args: ["environment", "local"]
- - name: "put"
- args: ["topology", "sourcefire"]
- - id: "metricConfig"
- className: "org.apache.commons.configuration.BaseConfiguration"
- configMethods:
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.graphite"
- - "${org.apache.metron.metrics.reporter.graphite}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.console"
- - "${org.apache.metron.metrics.reporter.console}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.reporter.jmx"
- - "${org.apache.metron.metrics.reporter.jmx}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.address"
- - "${org.apache.metron.metrics.graphite.address}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.graphite.port"
- - "${org.apache.metron.metrics.graphite.port}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.acks"
- - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.emits"
- - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryParserBolt.fails"
- - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
- - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
- - name: "setProperty"
- args:
- - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
- - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
-
-spouts:
- - id: "testingSpout"
- className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
- parallelism: 1
- configMethods:
- - name: "withFilename"
- args:
- - "SampleInput/SourcefireExampleOutput"
- - name: "withRepeating"
- args:
- - true
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.bolt.TelemetryParserBolt"
- configMethods:
- - name: "withMessageParser"
- args:
- - ref: "parser"
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - id: "indexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "sourcefire_index"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.dd.hh"
- - name: "withDocumentName"
- args:
- - "sourcefire_doc"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsBolt"
- className: "org.apache.metron.alerts.TelemetryAlertsBolt"
- configMethods:
- - name: "withIdentifier"
- args:
- - ref: "alertsIdentifier"
- - name: "withMaxCacheSize"
- args: [1000]
- - name: "withMaxTimeRetain"
- args: [3600]
- - name: "withAlertsAdapter"
- args:
- - ref: "alertsAdapter"
- - name: "withOutputFieldName"
- args: ["message"]
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "alertsIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "alert"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM.ww"
- - name: "withDocumentName"
- args:
- - "sourcefire_alert"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "errorIndexingBolt"
- className: "org.apache.metron.indexing.TelemetryIndexingBolt"
- configMethods:
- - name: "withIndexIP"
- args:
- - "${es.ip}"
- - name: "withIndexPort"
- args:
- - ${es.port}
- - name: "withClusterName"
- args:
- - "${es.clustername}"
- - name: "withIndexName"
- args:
- - "error"
- - name: "withIndexTimestamp"
- args:
- - "yyyy.MM"
- - name: "withDocumentName"
- args:
- - "sourcefire_error"
- - name: "withBulk"
- args:
- - 1
- - name: "withIndexAdapter"
- args:
- - ref: "indexAdapter"
- - name: "withMetricConfiguration"
- args:
- - ref: "metricConfig"
- - id: "geoEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "geoEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "hostEnrichmentBolt"
- className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
- configMethods:
- - name: "withEnrichment"
- args:
- - ref: "hostEnrichment"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
- - id: "joinBolt"
- className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt"
- configMethods:
- - name: "withEnrichments"
- args:
- - ref: "enrichments"
- - name: "withMaxCacheSize"
- args: [10000]
- - name: "withMaxTimeRetain"
- args: [10]
-
-streams:
- - name: "spout -> parser"
- from: "testingSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
- - name: "parser -> host"
- from: "parserBolt"
- to: "hostEnrichmentBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "parser -> geo"
- from: "parserBolt"
- to: "geoEnrichmentBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "parser -> join"
- from: "parserBolt"
- to: "joinBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "geo -> join"
- from: "geoEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "geo"
- type: FIELDS
- args: ["key"]
- - name: "host -> join"
- from: "hostEnrichmentBolt"
- to: "joinBolt"
- grouping:
- streamId: "host"
- type: FIELDS
- args: ["key"]
- - name: "join -> alerts"
- from: "joinBolt"
- to: "alertsBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "alerts -> alertsIndexing"
- from: "alertsBolt"
- to: "alertsIndexingBolt"
- grouping:
- streamId: "message"
- type: SHUFFLE
- - name: "join -> indexing"
- from: "joinBolt"
- to: "indexingBolt"
- grouping:
- streamId: "message"
- type: FIELDS
- args: ["key"]
- - name: "parser -> errors"
- from: "parserBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "indexing -> errors"
- from: "indexingBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
- - name: "alerts -> errors"
- from: "alertsBolt"
- to: "errorIndexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE