You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/02/22 18:33:41 UTC
[2/4] incubator-metron git commit: METRON-41 Integrate Apache Rat to
Audit Source Code Licensing (nickwallen via cestella) closes
apache/incubator-metron#27
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-EnrichmentAdapters/pom.xml.versionsBackup
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/pom.xml.versionsBackup b/metron-streaming/Metron-EnrichmentAdapters/pom.xml.versionsBackup
deleted file mode 100644
index 6c54dfc..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/pom.xml.versionsBackup
+++ /dev/null
@@ -1,136 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?><!-- Licensed to the Apache Software
- Foundation (ASF) under one or more contributor license agreements. See the
- NOTICE file distributed with this work for additional information regarding
- copyright ownership. The ASF licenses this file to You under the Apache License,
- Version 2.0 (the "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software distributed
- under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
- OR CONDITIONS OF ANY KIND, either express or implied. See the License for
- the specific language governing permissions and limitations under the License. -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-Streaming</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
- <artifactId>OpenSOC-EnrichmentAdapters</artifactId>
-
- <properties>
- <opensoc.common.version>0.0.1-SNAPSHOT</opensoc.common.version>
- <mysql.version>5.1.31</mysql.version>
- <slf4j.version>1.7.7</slf4j.version>
- <hbase.client.version>0.96.1-hadoop2</hbase.client.version>
- <storm.hdfs.version>0.1.2</storm.hdfs.version>
- <hadoop.version>2.2.0</hadoop.version>
- <storm.version>0.9.1-incubating</storm.version>
- <guava.version>17.0</guava.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-Common</artifactId>
- <version>${opensoc.common.version}</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>${slf4j.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>mysql</groupId>
- <artifactId>mysql-connector-java</artifactId>
- <version>${mysql.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${hbase.client.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- <version>${storm.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>3.8.2</version>
- </dependency>
- <dependency>
- <groupId>commons-validator</groupId>
- <artifactId>commons-validator</artifactId>
- <version>1.4.0</version>
- </dependency>
-
-
- </dependencies>
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <systemProperties>
- <property>
- <name>mode</name>
- <value>local</value>
- </property>
- </systemProperties>
- </configuration>
- </plugin>
- <!-- Normally, dependency report takes time, skip it -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- <version>2.7</version>
-
- <configuration>
- <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>emma-maven-plugin</artifactId>
- <version>1.0-alpha-3</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-pmd-plugin</artifactId>
- <configuration>
- <targetJdk>1.7</targetJdk>
- </configuration>
- </plugin>
- </plugins>
- </reporting>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-EnrichmentAdapters/readme.md
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/readme.md b/metron-streaming/Metron-EnrichmentAdapters/readme.md
deleted file mode 100644
index 5ae1f5b..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/readme.md
+++ /dev/null
@@ -1,125 +0,0 @@
-#Metron-Enrichments
-
-##Module Description
-
-This module enables enrichment of message metafields with additional information from various enrichment sources. Currently there is only a limited number of enrichments available, but this is an extensible framework that can be extended with additional enrichments. Enrichments currently available are geo, whois, hosts, and CIF.
-
-##Message Format
-
-Enrichment bolts are designed to go after the parser bolts. Parser bolts will parse the telemetry, taking it from its native format and producing a standard JSON that would look like so:
-
-```json
-{
-"message":
-{"ip_src_addr": xxxx,
-"ip_dst_addr": xxxx,
-"ip_src_port": xxxx,
-"ip_dst_port": xxxx,
-"protocol": xxxx,
-"additional-field 1": xxx,
-}
-
-}
-```
-
-A single enrichment bolt would enrich the message and produce a JSON enrichment and attach it to the message. Enrichments are stackable so multiple enrichments can be attached sequentially after a single parser bolt. Stacked enrichments would produce messages under the "enrichment" tag and attach it to the message like so:
-
-```json
-{
-"message":
-{"ip_src_addr": xxxx,
-"ip_dst_addr": xxxx,
-"ip_src_port": xxxx,
-"ip_dst_port": xxxx,
-"protocol": xxxx,
-"additional-field 1": xxxx,
-},
-"enrichment" : {"geo": xxxx, "whois": xxxx, "hosts": xxxxx, "CIF": "xxxxx"}
-
-}
-```
-
-##Enrichment Sources
-
-Each enrichment has to have an anrichment source which can serve as a lookup table for enriching relevant message fields. In order to minimize the use of additional platforms and tools we primarily try to rely on HBase as much as possible to store the enrichment information for lookup by key. In order to use Hbase we have to pre-process the enrichment feeds for bulk-loading into HBase with specific key format optimized for retrieval as well as utilize caches within the enrichment bolts to be able to provide enrichments real-time. Our wiki contains information on how to setup the environment, pre-process feeds, and plug in the enrichment sources.
-
-##Enrichment Bolt
-
-The enrichment bolt is designed to be extensible to be re-used for all kinds of enrichment processes. The bolt signature for declaration in a storm topology is as follows:
-
-
-
-```
-GenericEnrichmentBolt geo_enrichment = new GenericEnrichmentBolt()
-.withEnrichmentTag(
-config.getString("bolt.enrichment.geo.enrichment_tag"))
-.withAdapter(geo_adapter)
-.withMaxTimeRetain(
-config.getInt("bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES"))
-.withMaxCacheSize(
-config.getInt("bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM"))
-.withKeys(geo_keys).withMetricConfiguration(config);
-
-```
-
-EnrichmentTag - Name of the enrichment (geo, whois, hosts, etc)
-Keys - Keys which this enrichment is able to enrich (hosts field for hosts enrichment, source_ip, dest_ip, for geo enrichment, etc)
-MaxTimeToRetain & MaxCacheSize - define the caching policy of the enrichment bolt
-Adapter - which adapter to use with the enrichment bolt instance
-
-###Geo Adapter
-Geo adapter is able to do geo enrichment on hosts and destination IPs. The open source verison of the geo adapter uses the free Geo feeds from MaxMind. The format of these feeds does not easily lend itself to a no-sql DB so this adapter is designed to work with mySql. But it is extensible enough to be made work with a variety of other back ends.
-
-The signature of a geo adapter is as follows;
-
-```
-GeoMysqlAdapter geo_adapter = new GeoMysqlAdapter(
-config.getString("mysql.ip"), config.getInt("mysql.port"),
-config.getString("mysql.username"),
-config.getString("mysql.password"),
-config.getString("bolt.enrichment.geo.adapter.table"));
-
-```
-
-###Hosts Adapter
-The hosts adapter is designed to enrich message format with the static host information that can be read from a standard text file. This adapter is intended for use with a network crawling script that can identify all customer assets and place them in a text file. For example, this script would identify all workstations, printers, appliantces, etc. Then if any of these assets are seen in the telemetry messages flowing through the adapter this enrichment would fire and the relevant known information about a host would be attached. We are currently working on porting this adapter to work with HBase, but this work is not ready yet. The known hosts file is located under the /etc/whitelists config directory of Metron.
-
-The signature of the hosts adapter is as follows:
-
-```
-Map<String, JSONObject> known_hosts = SettingsLoader
-.loadKnownHosts(hosts_path);
-
-HostFromPropertiesFileAdapter host_adapter = new HostFromPropertiesFileAdapter(
-known_hosts);
-
-```
-* The source and dest ips refer to the name of the message JSON key where the host information is located
-
-###Whois Adapter
-Whois adapter enriches the host name with additional whois information obtained from our proprietary Cisco feed. The enricher itself is provided in this open source distribution, but the feed is not. You have to have your own feed in order to use it. Alternatively, you can contact us for providing you with this feed, but we would have to charge you a fee (we can't distribute it for free). The implemetation of the whois enrichment we provide works with HBase
-
-The signature of the whois adapter is as follows:
-
-```
-
-EnrichmentAdapter whois_adapter = new WhoisHBaseAdapter(
-config.getString("bolt.enrichment.whois.hbase.table.name"),
-config.getString("kafka.zk.list"),
-config.getString("kafka.zk.port"));
-```
-
-###CIF Adapter
-CIF adapter is designed to take in CIF feeds and cross-reference them against every message processed by Storm. If there is a hit then the relevant information is attached to the message.
-
-The signature of the CIF adapter is as follows:
-
-```
-CIFHbaseAdapter = new CIFHbaseAdapter(config
-.getString("kafka.zk.list"), config
-.getString("kafka.zk.port"), config
-.getString("bolt.enrichment.cif.tablename")))
-```
-
-##Stacking Enrichments
-Enrichments can be stacked. By default each enrichment bolt listens on the "message" stream. In order to create and stack enrichment bolts create a new bolt and instantiate the appropariate adapter. You can look at our sample topologies to see how enrichments can be stacked
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-FlumeConfigs/SampleFlumeConf.rtf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-FlumeConfigs/SampleFlumeConf.rtf b/metron-streaming/Metron-FlumeConfigs/SampleFlumeConf.rtf
deleted file mode 100644
index 728f08a..0000000
--- a/metron-streaming/Metron-FlumeConfigs/SampleFlumeConf.rtf
+++ /dev/null
@@ -1,43 +0,0 @@
-{\rtf1\ansi\ansicpg1252\cocoartf1265\cocoasubrtf210
-{\fonttbl\f0\fnil\fcharset0 Menlo-Regular;}
-{\colortbl;\red255\green255\blue255;}
-\margl1440\margr1440\vieww10800\viewh8400\viewkind0
-\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural
-
-\f0\fs22 \cf0 \CocoaLigature0 #agent section\
-flumesf1.sources = s\
-flumesf1.channels = c\
-flumesf1.sinks = r\
-\
-#source section\
-flumesf1.sources.s.type = syslogUdp\
-flumesf1.sources.s.port = 21001\
-flumesf1.sources.s.host = 0.0.0.0\
-flumesf1.sources.s.channels = c\
-\
-flumesf1.sinks.r.type = org.apache.flume.plugins.KafkaSink\
-\
-flumesf1.sinks.r.metadata.broker.list=dn01:9092,dn02:9092,dn03:9092,dn04:9092,dn08:9092,dn09:9092,dn10:9092\
-\
-#flumesf1.sinks.r.partition.key=0\
-#flumesf1.sinks.r.partitioner.class=org.apache.flume.plugins.SinglePartition\
-flumesf1.sinks.r.serializer.class=kafka.serializer.StringEncoder\
-flumesf1.sinks.r.request.required.acks=0\
-flumesf1.sinks.r.max.message.size=1000000\
-flumesf1.sinks.r.flumesf1.type=sync\
-flumesf1.sinks.r.custom.encoding=UTF-8\
-flumesf1.sinks.r.custom.topic.name=sourcefire_raw\
-\
-\
-\
-flumesf1.sinks.a.type = file_roll\
-flumesf1.sinks.a.channel = c\
-flumesf1.sinks.a.sink.directory = /tmp/flumesf1/\
-\
-\
-#Specify the channel the sink should use\
-flumesf1.sinks.r.channel = c\
-\
-# Each channel's type is defined.\
-flumesf1.channels.c.type = memory\
-flumesf1.channels.c.capacity = 1000}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-Indexing/README.md
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Indexing/README.md b/metron-streaming/Metron-Indexing/README.md
new file mode 100644
index 0000000..50d32e4
--- /dev/null
+++ b/metron-streaming/Metron-Indexing/README.md
@@ -0,0 +1,61 @@
+#Metron-Indexing
+
+##Module Description
+
+This module provides the indexing capability to Metron components. The primary indexing engine for now is Elastic Search, but Solr may be supported at some point in the future as well. There are three types of messages that are commonly indexed in Metron topologies: messages, alerts, and errors. Messages are telemetry messages parsed by the parser bolt. Alerts are alerts generated by the alerts bolt. Errors are an optional feature where each Metron bolt in addition to outputting errors in the log file will also index them for immediate analysis.
+
+###Index bolt
+
+The signature of the index bolt is as follows:
+
+```
+TelemetryIndexingBolt indexing_bolt = new TelemetryIndexingBolt()
+.withIndexIP(config.getString("es.ip"))
+.withIndexPort(config.getInt("es.port"))
+.withClusterName(config.getString("es.clustername"))
+.withIndexName(
+config.getString("bolt.error.indexing.indexname"))
+.withDocumentName(
+config.getString("bolt.error.indexing.documentname"))
+.withBulk(config.getInt("bolt.error.indexing.bulk"))
+.withIndexAdapter(adapter)
+.withMetricConfiguration(config);
+
+```
+
+###IndexAdapters
+
+*org.apache.metron.indexing.adapters.ESBaseBulkAdapter - bulk ingest messages into Elastic Search
+*org.apache.metron.indexing.adapters.ESBaseBulkRotatingAdapter - does everything adapter above does, but is able to rotate the index names based on size
+*org.apache.metron.indexing.adapters.ESTimedBulkRotatingAdapter - does everything adapter above does, but is able to rotate the index names based on size and time
+*org.apache.metron.indexing.adapters.SolrAdapter - currently under development
+
+/etc/ directory contains all environment-related configs
+
+##Sample Input and Generator Spout
+
+The sample input for topologies provided in this release was checked in here:
+
+```
+https://github.com/apache/incubator-metron-streaming/tree/master/Metron-Topologies/src/main/resources/SampleInput
+```
+
+We provide a generator spout that is able to drive these topologies. In production we run with the kafka spout, but for documentation on that please reference the Storm project documentation
+
+The generator spout comes with the following signature:
+
+```
+GenericInternalTestSpout testSpout = new GenericInternalTestSpout()
+.withFilename(test_file_path).withRepeating(
+config.getBoolean("spout.test.parallelism.repeat"));
+```
+
+* the repeat variable defines if the generator spout will loop through the input or stop once it gets to the end of file
+
+###Additional Storm Bolts
+In addition to custom bolts developed for Metron we utilize standard bolts and spouts included with the Storm release. We will not provide documentation for these spouts and bolts since they are provided as part of Storm. These spouts bolts are:
+
+* KafkaSpout
+* KafkaBolt
+* HDFSBolt
+* HBaseBolt
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-Indexing/pom.xml.versionsBackup
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Indexing/pom.xml.versionsBackup b/metron-streaming/Metron-Indexing/pom.xml.versionsBackup
deleted file mode 100644
index 11bf51e..0000000
--- a/metron-streaming/Metron-Indexing/pom.xml.versionsBackup
+++ /dev/null
@@ -1,91 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?><!-- Licensed to the Apache Software
- Foundation (ASF) under one or more contributor license agreements. See the
- NOTICE file distributed with this work for additional information regarding
- copyright ownership. The ASF licenses this file to You under the Apache License,
- Version 2.0 (the "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software distributed
- under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
- OR CONDITIONS OF ANY KIND, either express or implied. See the License for
- the specific language governing permissions and limitations under the License. -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-Streaming</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
- <artifactId>OpenSOC-Indexing</artifactId>
- <properties>
- <opensoc.common.version>0.0.1-SNAPSHOT</opensoc.common.version>
- <storm.version>0.9.1-incubating</storm.version>
- <elastic.search.version>1.2.1</elastic.search.version>
- <http.client.version>4.3.4</http.client.version>
- <jsonsimple.version>1.1.1</jsonsimple.version>
- </properties>
- <dependencies>
-
- <dependency>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-Common</artifactId>
- <version>${opensoc.common.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- <version>${storm.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>${elastic.search.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>${http.client.version}</version>
- </dependency>
- <dependency>
- <groupId>com.googlecode.json-simple</groupId>
- <artifactId>json-simple</artifactId>
- <version>${jsonsimple.version}</version>
- </dependency>
-
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- <version>1.9</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>3.8.2</version>
- </dependency>
-
- </dependencies>
-<reporting>
- <plugins>
- <!-- Normally, dependency report takes time, skip it -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- <version>2.7</version>
-
- <configuration>
- <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>emma-maven-plugin</artifactId>
- <version>1.0-alpha-3</version>
- <inherited>true</inherited>
- </plugin>
- </plugins>
- </reporting>
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-Indexing/readme.md
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Indexing/readme.md b/metron-streaming/Metron-Indexing/readme.md
deleted file mode 100644
index 4e81abd..0000000
--- a/metron-streaming/Metron-Indexing/readme.md
+++ /dev/null
@@ -1,61 +0,0 @@
-#Metron-Indexing
-
-##Module Description
-
-This module provides the indexing capability to Metron components. The primary indexing engine for now is Elastic Search, but Solr may be supported at some point in the future as well. There are three types of messages that are commonly indexed in Metron topologies: messages, alerts, and errors. Messages are telemetry messages parsed by the parser bolt. Alerts are alerts generated by the alerts bolt. Errors are an optional feature where each Metron bolt in addition to outputting errors in the log file will also index them for immediate analysis.
-
-###Index bolt
-
-The signature of the index bolt is as follows:
-
-```
-TelemetryIndexingBolt indexing_bolt = new TelemetryIndexingBolt()
-.withIndexIP(config.getString("es.ip"))
-.withIndexPort(config.getInt("es.port"))
-.withClusterName(config.getString("es.clustername"))
-.withIndexName(
-config.getString("bolt.error.indexing.indexname"))
-.withDocumentName(
-config.getString("bolt.error.indexing.documentname"))
-.withBulk(config.getInt("bolt.error.indexing.bulk"))
-.withIndexAdapter(adapter)
-.withMetricConfiguration(config);
-
-```
-
-###IndexAdapters
-
-*org.apache.metron.indexing.adapters.ESBaseBulkAdapter - bulk ingest messages into Elastic Search
-*org.apache.metron.indexing.adapters.ESBaseBulkRotatingAdapter - does everything adapter above does, but is able to rotate the index names based on size
-*org.apache.metron.indexing.adapters.ESTimedBulkRotatingAdapter - does everything adapter above does, but is able to rotate the index names based on size and time
-*org.apache.metron.indexing.adapters.SolrAdapter - currently under development
-
-/etc/ directory contains all environment-related configs
-
-##Sample Input and Generator Spout
-
-The sample input for topologies provided in this release was checked in here:
-
-```
-https://github.com/apache/incubator-metron-streaming/tree/master/Metron-Topologies/src/main/resources/SampleInput
-```
-
-We provide a generator spout that is able to drive these topologies. In production we run with the kafka spout, but for documentation on that please reference the Storm project documentation
-
-The generator spout comes with the following signature:
-
-```
-GenericInternalTestSpout testSpout = new GenericInternalTestSpout()
-.withFilename(test_file_path).withRepeating(
-config.getBoolean("spout.test.parallelism.repeat"));
-```
-
-* the repeat variable defines if the generator spout will loop through the input or stop once it gets to the end of file
-
-###Additional Storm Bolts
-In addition to custom bolts developed for Metron we utilize standard bolts and spouts included with the Storm release. We will not provide documentation for these spouts and bolts since they are provided as part of Storm. These spouts bolts are:
-
-* KafkaSpout
-* KafkaBolt
-* HDFSBolt
-* HBaseBolt
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-MessageParsers/README.md
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/README.md b/metron-streaming/Metron-MessageParsers/README.md
new file mode 100644
index 0000000..509f02e
--- /dev/null
+++ b/metron-streaming/Metron-MessageParsers/README.md
@@ -0,0 +1,82 @@
+#Metron-Parsers
+
+##Module Description
+
+This module provides a list of parsers that can be used with the Metron framework. There are two types of parsers. First type is a Java parser. This kind of parser is optimized for speed and performance and is built for use with higher velicity topologies. These parsers are not easily modifiable and in order to make changes to them the entire topology need to be recompiled. The second type of parser provided with the system is a Grok parser. This type of parser is primarily designed for lower-velocity topologies or for quickly standing up a parser for a new telemetry before a permanent Java parser can be written for it.
+
+##Message Format
+
+All Metron messages follow a specific format in order to ingest a message. If a message does not conform to this format it will be dropped and put onto an error queue for further examination. The message must be of a JSON format and must have a JSON tag message like so:
+
+```
+{"message" : message content}
+
+```
+
+Where appropriate there is also a standardization around the 5-tuple JSON fields. This is done so the topology correlation engine further down stream can correlate messages from different topologies by these fields. We are currently working on expanding the message standardization beyond these fields, but this feature is not yet availabe. The standard field names are as follows:
+
+* ip_src_addr: layer 3 source IP
+* ip_dst_addr: layer 3 dest IP
+* ip_src_port: layer 4 source port
+* ip_dst_port: layer 4 dest port
+* protocol: layer 4 protocol
+* timestamp (epoch)
+* original_string: A human friendly string representation of the message
+
+The timestamp and original_string fields are madatory. The remaining standard fields are optional. If any of the optional fields are not applicable then the field should be left out of the JSON.
+
+So putting it all together a typical Metron message with all 5-tuple fields present would look like the following:
+
+```json
+{
+"message":
+{"ip_src_addr": xxxx,
+"ip_dst_addr": xxxx,
+"ip_src_port": xxxx,
+"ip_dst_port": xxxx,
+"protocol": xxxx,
+"original_string": xxx,
+"additional-field 1": xxx,
+}
+
+}
+```
+
+##Parser Bolt
+
+The Metron parser bolt is a standard bolt, which can be extended with multiple Java and Grok parser adapter for parsing different topology messages. The bolt signature for declaration in a storm topology is as follows:
+
+```
+AbstractParserBolt parser_bolt = new TelemetryParserBolt()
+.withMessageParser(parser)
+.withMessageFilter(new GenericMessageFilter())
+.withMetricConfig(config);
+
+```
+
+Metric Config - optional argument for exporting custom metrics to graphite. If set to null no metrics will be exported. If set, then a list of metrics defined in the metrics.conf file of each topology will define will metrics are exported and how often.
+
+Message Filter - a filter defining which messages can be dropped. This feature is only present in the Java paerer adapters
+
+Message Parser - defines the parser adapter to be used for a topology
+
+##Parser Adapters
+
+Parser adapters are loaded dynamically in each Metron topology. They are defined in topology.conf in the configuration item bolt.parser.adapter
+
+###Java Parser Adapters
+Java parser adapters are indended for higher-velocity topologies and are not easily changed or extended. As the adoption of Metron continues we plan on extending our library of Java adapters to process more log formats. As of this moment the Java adapters included with Metron are:
+
+* org.apache.metron.parsing.parsers.BasicIseParser : Parse ISE messages
+* org.apache.metron.parsing.parsers.BasicBroParser : Parse Bro messages
+* org.apache.metron.parsing.parsers.BasicSourcefireParser : Parse Sourcefire messages
+* org.apache.metron.parsing.parsers.BasicLancopeParser : Parse Lancope messages
+
+###Grok Parser Adapters
+Grok parser adapters are designed primarly for someone who is not a Java coder for quickly standing up a parser adapter for lower velocity topologies. Grok relies on Regex for message parsing, which is much slower than purpose-built Java parsers, but is more extensible. Grok parsers are defined via a config file and the topplogy does not need to be recombiled in order to make changes to them. An example of a Grok perser is:
+
+* org.apache.metron.parsing.parsers.GrokSourcefireParser
+
+For more information on the Grok project please refer to the following link:
+
+https://github.com/thekrakken/java-grok
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-MessageParsers/pom.xml.versionsBackup
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/pom.xml.versionsBackup b/metron-streaming/Metron-MessageParsers/pom.xml.versionsBackup
deleted file mode 100644
index ef2d97d..0000000
--- a/metron-streaming/Metron-MessageParsers/pom.xml.versionsBackup
+++ /dev/null
@@ -1,74 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?><!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-Streaming</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
- <artifactId>OpenSOC-MessageParsers</artifactId>
- <properties>
- <opensoc.common.version>0.0.1-SNAPSHOT</opensoc.common.version>
- <storm.version>0.9.1-incubating</storm.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-Common</artifactId>
- <version>${opensoc.common.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- <version>${storm.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>3.8.2</version>
- </dependency>
- </dependencies>
-<reporting>
- <plugins>
- <!-- Normally, dependency report takes time, skip it -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- <version>2.7</version>
-
- <configuration>
- <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>emma-maven-plugin</artifactId>
- <version>1.0-alpha-3</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-pmd-plugin</artifactId>
- <configuration>
- <targetJdk>1.7</targetJdk>
- </configuration>
- </plugin>
- </plugins>
- </reporting>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-MessageParsers/readme.md
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/readme.md b/metron-streaming/Metron-MessageParsers/readme.md
deleted file mode 100644
index 509f02e..0000000
--- a/metron-streaming/Metron-MessageParsers/readme.md
+++ /dev/null
@@ -1,82 +0,0 @@
-#Metron-Parsers
-
-##Module Description
-
-This module provides a list of parsers that can be used with the Metron framework. There are two types of parsers. First type is a Java parser. This kind of parser is optimized for speed and performance and is built for use with higher velicity topologies. These parsers are not easily modifiable and in order to make changes to them the entire topology need to be recompiled. The second type of parser provided with the system is a Grok parser. This type of parser is primarily designed for lower-velocity topologies or for quickly standing up a parser for a new telemetry before a permanent Java parser can be written for it.
-
-##Message Format
-
-All Metron messages follow a specific format in order to ingest a message. If a message does not conform to this format it will be dropped and put onto an error queue for further examination. The message must be of a JSON format and must have a JSON tag message like so:
-
-```
-{"message" : message content}
-
-```
-
-Where appropriate there is also a standardization around the 5-tuple JSON fields. This is done so the topology correlation engine further down stream can correlate messages from different topologies by these fields. We are currently working on expanding the message standardization beyond these fields, but this feature is not yet availabe. The standard field names are as follows:
-
-* ip_src_addr: layer 3 source IP
-* ip_dst_addr: layer 3 dest IP
-* ip_src_port: layer 4 source port
-* ip_dst_port: layer 4 dest port
-* protocol: layer 4 protocol
-* timestamp (epoch)
-* original_string: A human friendly string representation of the message
-
-The timestamp and original_string fields are madatory. The remaining standard fields are optional. If any of the optional fields are not applicable then the field should be left out of the JSON.
-
-So putting it all together a typical Metron message with all 5-tuple fields present would look like the following:
-
-```json
-{
-"message":
-{"ip_src_addr": xxxx,
-"ip_dst_addr": xxxx,
-"ip_src_port": xxxx,
-"ip_dst_port": xxxx,
-"protocol": xxxx,
-"original_string": xxx,
-"additional-field 1": xxx,
-}
-
-}
-```
-
-##Parser Bolt
-
-The Metron parser bolt is a standard bolt, which can be extended with multiple Java and Grok parser adapter for parsing different topology messages. The bolt signature for declaration in a storm topology is as follows:
-
-```
-AbstractParserBolt parser_bolt = new TelemetryParserBolt()
-.withMessageParser(parser)
-.withMessageFilter(new GenericMessageFilter())
-.withMetricConfig(config);
-
-```
-
-Metric Config - optional argument for exporting custom metrics to graphite. If set to null no metrics will be exported. If set, then a list of metrics defined in the metrics.conf file of each topology will define will metrics are exported and how often.
-
-Message Filter - a filter defining which messages can be dropped. This feature is only present in the Java paerer adapters
-
-Message Parser - defines the parser adapter to be used for a topology
-
-##Parser Adapters
-
-Parser adapters are loaded dynamically in each Metron topology. They are defined in topology.conf in the configuration item bolt.parser.adapter
-
-###Java Parser Adapters
-Java parser adapters are indended for higher-velocity topologies and are not easily changed or extended. As the adoption of Metron continues we plan on extending our library of Java adapters to process more log formats. As of this moment the Java adapters included with Metron are:
-
-* org.apache.metron.parsing.parsers.BasicIseParser : Parse ISE messages
-* org.apache.metron.parsing.parsers.BasicBroParser : Parse Bro messages
-* org.apache.metron.parsing.parsers.BasicSourcefireParser : Parse Sourcefire messages
-* org.apache.metron.parsing.parsers.BasicLancopeParser : Parse Lancope messages
-
-###Grok Parser Adapters
-Grok parser adapters are designed primarly for someone who is not a Java coder for quickly standing up a parser adapter for lower velocity topologies. Grok relies on Regex for message parsing, which is much slower than purpose-built Java parsers, but is more extensible. Grok parsers are defined via a config file and the topplogy does not need to be recombiled in order to make changes to them. An example of a Grok perser is:
-
-* org.apache.metron.parsing.parsers.GrokSourcefireParser
-
-For more information on the Grok project please refer to the following link:
-
-https://github.com/thekrakken/java-grok
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicFireEyeParserTest.config
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicFireEyeParserTest.config b/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicFireEyeParserTest.config
index 8073cec..4b4c648 100644
--- a/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicFireEyeParserTest.config
+++ b/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicFireEyeParserTest.config
@@ -1,2 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
#BasicFireEyeParserTestConfig
logFile=src/test/resources/FireEyeParserTest.log
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicIseParserTest.config
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicIseParserTest.config b/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicIseParserTest.config
index ac158a5..d57dace 100644
--- a/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicIseParserTest.config
+++ b/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicIseParserTest.config
@@ -1,2 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
#IseParserTestConfig
logFile=src/test/resources/IseParserTest.log
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicLancopeParserTest.config
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicLancopeParserTest.config b/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicLancopeParserTest.config
index edafc56..bd32770 100644
--- a/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicLancopeParserTest.config
+++ b/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicLancopeParserTest.config
@@ -1,2 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
#LancopeParserTestConfig
logFile=src/test/resources/LancopeParserTest.log
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicPaloAltoFirewallParserTest.config
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicPaloAltoFirewallParserTest.config b/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicPaloAltoFirewallParserTest.config
index 613c314..61b649e 100644
--- a/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicPaloAltoFirewallParserTest.config
+++ b/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicPaloAltoFirewallParserTest.config
@@ -1,2 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
#BasicFireEyeParserTestConfig
logFile=src/test/resources/PaloAltoFirewallParserTest.log
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicSourcefireParserTest.config
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicSourcefireParserTest.config b/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicSourcefireParserTest.config
index 556a54c..685b0fd 100644
--- a/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicSourcefireParserTest.config
+++ b/metron-streaming/Metron-MessageParsers/src/test/resources/config/BasicSourcefireParserTest.config
@@ -1,2 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
#BasicSourceFileParserTestConfig
logFile=src/test/resources/SourceFireTest.log
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-MessageParsers/src/test/resources/config/BroParserTest.config
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/test/resources/config/BroParserTest.config b/metron-streaming/Metron-MessageParsers/src/test/resources/config/BroParserTest.config
index c50743c..0dcfc71 100644
--- a/metron-streaming/Metron-MessageParsers/src/test/resources/config/BroParserTest.config
+++ b/metron-streaming/Metron-MessageParsers/src/test/resources/config/BroParserTest.config
@@ -1,2 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
#BroParserTestConfig
logFile=src/test/resources/BroParserTest.log
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-MessageParsers/src/test/resources/config/GrokAsaParserTest.config
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/test/resources/config/GrokAsaParserTest.config b/metron-streaming/Metron-MessageParsers/src/test/resources/config/GrokAsaParserTest.config
index 2f41210..9dbc3b6 100644
--- a/metron-streaming/Metron-MessageParsers/src/test/resources/config/GrokAsaParserTest.config
+++ b/metron-streaming/Metron-MessageParsers/src/test/resources/config/GrokAsaParserTest.config
@@ -1,2 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
#GrokParserTestConfig
logFile=src/test/resources/GrokParserTest.log
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-Pcap_Service/pom.xml.versionsBackup
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Pcap_Service/pom.xml.versionsBackup b/metron-streaming/Metron-Pcap_Service/pom.xml.versionsBackup
deleted file mode 100644
index a400fe2..0000000
--- a/metron-streaming/Metron-Pcap_Service/pom.xml.versionsBackup
+++ /dev/null
@@ -1,268 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-Streaming</artifactId>
- <version>0.4BETA</version>
- </parent>
- <artifactId>OpenSOC-Pcap_Service</artifactId>
- <description>OpenSOC Pcap_Service</description>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <flume.version>1.4.0.2.0.6.0-76</flume.version>
- <hadoop.version>2.2.0.2.0.6.0-76</hadoop.version>
- <maven.compiler.source>${jdk.version}</maven.compiler.source>
- <maven.compiler.target>${jdk.version}</maven.compiler.target>
-
- <storm.version>0.9.2-incubating</storm.version>
- <kafka.version>0.8.0</kafka.version>
- <slf4j.version>1.7.5</slf4j.version>
- <zookeeper.version>3.4.5.2.0.6.0-76</zookeeper.version>
- <logger.version>1.2.15</logger.version>
-
- <storm-kafka.version>0.9.2-incubating</storm-kafka.version>
- <storm-hdfs.version>0.0.7-SNAPSHOT</storm-hdfs.version>
- <storm-hbase.version>0.0.5-SNAPSHOT</storm-hbase.version>
-
- <spring.integration.version>3.0.0.RELEASE</spring.integration.version>
- <spring.version>3.2.6.RELEASE</spring.version>
- <commons-fileupload.version>1.2.2</commons-fileupload.version>
- <commons-io.version>2.4</commons-io.version>
- <commons-configuration.version>1.10</commons-configuration.version>
- <commons-lang.version>2.6</commons-lang.version>
- <commons-collections.version>3.2.1</commons-collections.version>
- <commons-beanutils.version>1.8.3</commons-beanutils.version>
- <commons-jexl.version>2.1.1</commons-jexl.version>
-
-
- <junit.version>4.11</junit.version>
- <hamcrest.version>1.3</hamcrest.version>
- <mockito.version>1.9.5</mockito.version>
- <elastic-search.version>1.3.0</elastic-search.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.jboss.resteasy</groupId>
- <artifactId>jaxrs-api</artifactId>
- <version>3.0.4.Final</version>
- </dependency>
- <dependency>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-Common</artifactId>
- <version>${parent.version}</version>
- </dependency>
-
- <dependency>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils</artifactId>
- <version>${commons-beanutils.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-jexl</artifactId>
- <version>${commons-jexl.version}</version>
- </dependency>
-
- <dependency>
- <artifactId>commons-configuration</artifactId>
- <groupId>commons-configuration</groupId>
- <version>${commons-configuration.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>${junit.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-mockito</artifactId>
- <version>1.5</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-core</artifactId>
- <version>1.5</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <version>1.5</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- <version>2.3</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${global_hbase_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-testing-util</artifactId>
- <version>${global_hbase_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${global_hadoop_version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${global_hadoop_version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.springframework.integration</groupId>
- <artifactId>spring-integration-http</artifactId>
- <version>${spring.integration.version}</version>
- </dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-webmvc</artifactId>
- <version>${spring.version}</version>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>${logger.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jmx</groupId>
- <artifactId>jmxri</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jdmk</groupId>
- <artifactId>jmxtools</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.jms</groupId>
- <artifactId>jms</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
-
-
-
-
- <dependency>
- <groupId>org.jboss.resteasy</groupId>
- <artifactId>resteasy-jaxrs</artifactId>
- <version>3.0.1.Final</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.jboss.resteasy</groupId>
- <artifactId>resteasy-jaxb-provider</artifactId>
- <version>3.0.1.Final</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.jboss.resteasy</groupId>
- <artifactId>async-http-servlet-3.0</artifactId>
- <version>3.0.1.Final</version>
- <scope>compile</scope>
- </dependency>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
- <version>9.3.0.M0</version>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-servlet</artifactId>
- <version>9.3.0.M0</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
- <version>${global_slf4j_version}</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>${global_slf4j_version}</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>${global_slf4j_version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <archive>
- <manifest>
- <mainClass>com.opensoc.pcapservice.rest.PcapService</mainClass>
- </manifest>
- </archive>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id> <!-- this is used for inheritance merges -->
- <phase>package</phase> <!-- bind to the packaging phase -->
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-Pcap_Service/src/main/resources/config-definition-hbase.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Pcap_Service/src/main/resources/config-definition-hbase.xml b/metron-streaming/Metron-Pcap_Service/src/main/resources/config-definition-hbase.xml
index efe05e8..98ece42 100644
--- a/metron-streaming/Metron-Pcap_Service/src/main/resources/config-definition-hbase.xml
+++ b/metron-streaming/Metron-Pcap_Service/src/main/resources/config-definition-hbase.xml
@@ -1,5 +1,21 @@
<?xml version="1.0" encoding="ISO-8859-1" ?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
<configuration>
<header>
<result delimiterParsingDisabled="true" forceReloadCheck="true"></result>
@@ -15,20 +31,20 @@
</lookups>
</header>
<override>
- <!-- 1. properties from 'hbae-config.properties' are loaded first;
+ <!-- 1. properties from 'hbae-config.properties' are loaded first;
if a property is not present in this file, then it will search in the files in the order they are defined here.
2. 'refreshDelay' indicates the minimum delay in milliseconds between checks to see if the underlying file is changed.
- 3. 'config-optional' indicates this file is not required -->
-
+ 3. 'config-optional' indicates this file is not required -->
+
<properties fileName="${expr:System.getProperty('configPath')+'/hbase-config.properties'}" config-optional="true">
<reloadingStrategy refreshDelay="${expr:System.getProperty('configRefreshDelay')}"
config-class="org.apache.commons.configuration.reloading.FileChangedReloadingStrategy"/>
</properties>
-
+
<properties fileName="hbase-config-default.properties" config-optional="true">
<!-- <reloadingStrategy refreshDelay="${expr:System.getProperty('defaultConfigRefreshDelay')}"
config-class="org.apache.commons.configuration.reloading.FileChangedReloadingStrategy"/>
--> </properties>
-
+
</override>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-Topologies/README.md
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/README.md b/metron-streaming/Metron-Topologies/README.md
new file mode 100644
index 0000000..f19d910
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/README.md
@@ -0,0 +1,39 @@
+#Metron-Topologies
+
+#Module Description
+
+This module provides example topologies that show how to drive Metron modules and components. The sample topologies provided are to process PCAP, Ise, Lancope, and Bro telemetries
+
+##Launching Topologies
+
+We use Storm Flux to launch topologies, which are each described in a YAML file.
+
+```
+storm jar target/Metron-Topologies-0.1BETA.jar org.apache.storm.flux.Flux --local src/main/resources/Metron_Configs/topologies/bro/local.yaml --filter src/main/resources/Metron_Configs/etc/env/config.properties
+
+storm jar target/Metron-Topologies-0.1BETA.jar org.apache.storm.flux.Flux --remote src/main/resources/Metron_Configs/topologies/bro/remote.yaml --filter src/main/resources/Metron_Configs/etc/env/config.properties
+```
+
+Note that if you use `--local` it will run the topology in local mode, using test data. If you use `--remote` it will attempt to connect to and deploy to Storm Nimbus.
+
+Each topology's YAML files are responsible for either connecting to a real spout or enabling their own testing spout. This is the primary reason different `local.yaml` and `remote.yaml` files are provided for each topology.
+
+##Topology Configs
+
+The sample topologies provided use a specific directory structure. The example directory structure was checked in here:
+
+```
+https://github.com/apache/incubator-metron/tree/master/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs
+```
+
+Each topology has a `local.yaml` and a `remote.yaml` file to support local mode and remote mode, respectively.
+
+These topology configurations have variables that can be replaced by the `--filter` option to Flux. These variables are in `src/main/resources/OpenSOC_Configs/etc/env/config.properties`, and apply to:
+
+- Kafka
+- Elasticsearch
+- MySQL
+- Metrics
+- Bolt acks/emits/fails
+- Host enrichment
+- HDFS
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-Topologies/pom.xml.versionsBackup
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/pom.xml.versionsBackup b/metron-streaming/Metron-Topologies/pom.xml.versionsBackup
deleted file mode 100644
index 98ada2d..0000000
--- a/metron-streaming/Metron-Topologies/pom.xml.versionsBackup
+++ /dev/null
@@ -1,190 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?><!-- Licensed to the Apache Software
- Foundation (ASF) under one or more contributor license agreements. See the
- NOTICE file distributed with this work for additional information regarding
- copyright ownership. The ASF licenses this file to You under the Apache License,
- Version 2.0 (the "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software distributed
- under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
- OR CONDITIONS OF ANY KIND, either express or implied. See the License for
- the specific language governing permissions and limitations under the License. -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-Streaming</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
- <artifactId>OpenSOC-Topologies</artifactId>
- <description>OpenSOC Topologies</description>
-
- <properties>
- <opensoc.common.version>0.0.1-SNAPSHOT</opensoc.common.version>
- <opensoc.alerts.version>0.0.1-SNAPSHOT</opensoc.alerts.version>
- <enrichment.geo.version>0.0.1-SNAPSHOT</enrichment.geo.version>
- <enrichment.whois.version>0.0.1-SNAPSHOT</enrichment.whois.version>
- <geo.enrichment.version>0.0.1-SNAPSHOT</geo.enrichment.version>
- <parsers.version>0.0.1-SNAPSHOT</parsers.version>
- <indexing.version>0.0.1-SNAPSHOT</indexing.version>
- <storm.version>0.9.2-incubating</storm.version>
- <storm.hdfs.version>0.9.1.2.1.1.0-385</storm.hdfs.version>
- <hadoop.version>2.2.0</hadoop.version>
- <cli.version>20040117.000000</cli.version>
- <kafka.storm.version>0.9.2-incubating</kafka.storm.version>
- <cif.enrichment.version>0.0.1-SNAPSHOT</cif.enrichment.version>
- <lancope.enrichment.version>0.0.1-SNAPSHOT</lancope.enrichment.version>
- <commons.config.version>1.10</commons.config.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-Common</artifactId>
- <version>${opensoc.common.version}</version>
- </dependency>
- <dependency>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-Alerts</artifactId>
- <version>${opensoc.alerts.version}</version>
- </dependency>
- <dependency>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-EnrichmentAdapters</artifactId>
- <version>${enrichment.geo.version}</version>
- </dependency>
- <dependency>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-MessageParsers</artifactId>
- <version>${parsers.version}</version>
- </dependency>
- <dependency>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-Indexing</artifactId>
- <version>${indexing.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.8.2</artifactId>
- <version>0.8.1</version>
- <exclusions>
- <!--exclusion> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId>
- </exclusion -->
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- <version>${storm.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-kafka</artifactId>
- <version>${storm.version}</version>
-
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm</artifactId>
- <version>${storm.version}</version>
- <type>pom</type>
- <scope>provided</scope>
- </dependency>
-<dependency>
- <groupId>com.github.ptgoetz</groupId>
- <artifactId>storm-hbase</artifactId>
- <version>0.1.2</version>
-</dependency>
- <dependency>
- <groupId>com.github.ptgoetz</groupId>
- <artifactId>storm-hdfs</artifactId>
- <version>0.1.2</version>
- </dependency>
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- <version>${commons.config.version}</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>3.8.2</version>
- </dependency>
- </dependencies>
- <build>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <plugins>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>1.4</version>
- <configuration>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <artifactSet>
- <excludes>
- <exclude>storm:storm-core:*</exclude>
- <exclude>storm:storm-lib:*</exclude>
- <exclude>*slf4j*</exclude>
- </excludes>
- </artifactSet>
-
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resource>.yaml</resource>
- </transformer>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass></mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-Topologies/readme.md
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/readme.md b/metron-streaming/Metron-Topologies/readme.md
deleted file mode 100644
index f19d910..0000000
--- a/metron-streaming/Metron-Topologies/readme.md
+++ /dev/null
@@ -1,39 +0,0 @@
-#Metron-Topologies
-
-#Module Description
-
-This module provides example topologies that show how to drive Metron modules and components. The sample topologies provided are to process PCAP, Ise, Lancope, and Bro telemetries
-
-##Launching Topologies
-
-We use Storm Flux to launch topologies, which are each described in a YAML file.
-
-```
-storm jar target/Metron-Topologies-0.1BETA.jar org.apache.storm.flux.Flux --local src/main/resources/Metron_Configs/topologies/bro/local.yaml --filter src/main/resources/Metron_Configs/etc/env/config.properties
-
-storm jar target/Metron-Topologies-0.1BETA.jar org.apache.storm.flux.Flux --remote src/main/resources/Metron_Configs/topologies/bro/remote.yaml --filter src/main/resources/Metron_Configs/etc/env/config.properties
-```
-
-Note that if you use `--local` it will run the topology in local mode, using test data. If you use `--remote` it will attempt to connect to and deploy to Storm Nimbus.
-
-Each topology's YAML files are responsible for either connecting to a real spout or enabling their own testing spout. This is the primary reason different `local.yaml` and `remote.yaml` files are provided for each topology.
-
-##Topology Configs
-
-The sample topologies provided use a specific directory structure. The example directory structure was checked in here:
-
-```
-https://github.com/apache/incubator-metron/tree/master/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs
-```
-
-Each topology has a `local.yaml` and a `remote.yaml` file to support local mode and remote mode, respectively.
-
-These topology configurations have variables that can be replaced by the `--filter` option to Flux. These variables are in `src/main/resources/OpenSOC_Configs/etc/env/config.properties`, and apply to:
-
-- Kafka
-- Elasticsearch
-- MySQL
-- Metrics
-- Bolt acks/emits/fails
-- Host enrichment
-- HDFS
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/whitelists/known_hosts.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/whitelists/known_hosts.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/whitelists/known_hosts.conf
index d20b050..d5b9a33 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/whitelists/known_hosts.conf
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/whitelists/known_hosts.conf
@@ -1,4 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
10.1.128.236={"local":"YES", "type":"webserver", "asset_value" : "important"}
10.1.128.237={"local":"UNKNOWN", "type":"unknown", "asset_value" : "important"}
10.60.10.254={"local":"YES", "type":"printer", "asset_value" : "important"}
-10.0.2.15={"local":"YES", "type":"webserver", "asset_value" : "important"}
\ No newline at end of file
+10.0.2.15={"local":"YES", "type":"webserver", "asset_value" : "important"}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/Metron-Topologies/src/main/resources/TopologyConfigs_old/lancope.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/TopologyConfigs_old/lancope.conf b/metron-streaming/Metron-Topologies/src/main/resources/TopologyConfigs_old/lancope.conf
index c0487d2..454c7da 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/TopologyConfigs_old/lancope.conf
+++ b/metron-streaming/Metron-Topologies/src/main/resources/TopologyConfigs_old/lancope.conf
@@ -1,3 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
include = env/env_preprod.conf
include = env/metrics.conf
@@ -87,4 +105,4 @@ bolt.hdfs.fs.url=hdfs://nn1:8020
#Kafka Bolt
bolt.kafka.num.tasks=1
bolt.kafka.parallelism.hint=1
-bolt.kafka.topic=test_out
\ No newline at end of file
+bolt.kafka.topic=test_out
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/README.md
----------------------------------------------------------------------
diff --git a/metron-streaming/README.md b/metron-streaming/README.md
new file mode 100644
index 0000000..4b4df69
--- /dev/null
+++ b/metron-streaming/README.md
@@ -0,0 +1,30 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+ -->
+
+# Current Build
+
+The latest build of Metron-Streaming is 0.1BETA.
+
+We are still in the process of merging/porting additional features from our production code base into this open source release. This release will be followed by a number of additional beta releases until the port is complete. We will also work on getting additional documentation and user/developer guides to the community as soon as we can. At this time we offer no support for the beta software, but will try to respond to requests as promptly as we can.
+
+# Metron-Streaming
+
+Extensible set of Storm topologies and topology attributes for streaming, enriching, indexing, and storing telemetry in Hadoop. General information on Metron is available at https://metron.incubator.apache.org/
+
+# Documentation
+
+Please see documentation within each individual module for description and usage instructions. Sample topologies are provided under Metron_Topologies to get you started with the framework. We pre-assume knowledge of Hadoop, Storm, Kafka, and HBase.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/pom.xml b/metron-streaming/pom.xml
index 1b7cdd6..47ab2cb 100644
--- a/metron-streaming/pom.xml
+++ b/metron-streaming/pom.xml
@@ -1,15 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software
- Foundation (ASF) under one or more contributor license agreements. See the
- NOTICE file distributed with this work for additional information regarding
- copyright ownership. The ASF licenses this file to You under the Apache License,
- Version 2.0 (the "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software distributed
- under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
- OR CONDITIONS OF ANY KIND, either express or implied. See the License for
- the specific language governing permissions and limitations under the License.
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
@@ -60,8 +60,6 @@
</properties>
</developer>
</developers>
-
-
<modules>
<module>Metron-Common</module>
<module>Metron-EnrichmentAdapters</module>
@@ -80,7 +78,6 @@
<scope>test</scope>
</dependency>
</dependencies>
-
<build>
<plugins>
<plugin>
@@ -123,9 +120,23 @@
<aggregate>true</aggregate>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>0.11</version>
+ <configuration>
+ <excludes>
+ <exclude>**/README.md</exclude>
+ <exclude>**/*.json</exclude>
+ <exclude>**/*.log</exclude>
+ <exclude>**/src/main/resources/patterns/**</exclude>
+ <exclude>**/src/main/resources/SampleInput/**</exclude>
+ <exclude>**/dependency-reduced-pom.xml</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</build>
-
<reporting>
<plugins>
<plugin>
@@ -147,12 +158,10 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-project-info-reports-plugin</artifactId>
<version>2.7</version>
-
<configuration>
<dependencyLocationsEnabled>false</dependencyLocationsEnabled>
</configuration>
</plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-pmd-plugin</artifactId>
@@ -170,7 +179,6 @@
</plugins>
</reporting>
<repositories>
-
<repository>
<id>clojars.org</id>
<url>http://clojars.org/repo</url>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/pom.xml.versionsBackup
----------------------------------------------------------------------
diff --git a/metron-streaming/pom.xml.versionsBackup b/metron-streaming/pom.xml.versionsBackup
deleted file mode 100644
index 7302ae6..0000000
--- a/metron-streaming/pom.xml.versionsBackup
+++ /dev/null
@@ -1,104 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?><!-- Licensed to the Apache Software
- Foundation (ASF) under one or more contributor license agreements. See the
- NOTICE file distributed with this work for additional information regarding
- copyright ownership. The ASF licenses this file to You under the Apache License,
- Version 2.0 (the "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software distributed
- under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
- OR CONDITIONS OF ANY KIND, either express or implied. See the License for
- the specific language governing permissions and limitations under the License. -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>com.opensoc</groupId>
- <artifactId>OpenSOC-Streaming</artifactId>
- <version>BETA_0.2</version>
- <packaging>pom</packaging>
- <name>OpenSOC-Streaming</name>
- <description>Stream analytics for OpenSOC</description>
- <url>www.getopensoc.com</url>
- <properties>
- <twitter>@ProjectOpenSOC</twitter>
- <global_version>BETA_0.2</global_version>
- </properties>
- <licenses>
- <license>
- <name>The Apache Software License, Version 2.0</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
- <distribution>repo</distribution>
- </license>
- </licenses>
- <developers>
- <developer>
- <id>jsirota</id>
- <name>James Sirota</name>
- <email>jsirota@cisco.com</email>
- <properties>
- <twitter>@JamesSirota</twitter>
- <blog>medium.com/@JamesSirota</blog>
- </properties>
- </developer>
- </developers>
-
- <modules>
- <module>OpenSOC-Common</module>
- <module>OpenSOC-EnrichmentAdapters</module>
- <module>OpenSOC-MessageParsers</module>
- <module>OpenSOC-Indexing</module>
- <module>OpenSOC-Alerts</module>
- <module>OpenSOC-DataLoads</module>
- <module>OpenSOC-Topologies</module>
- </modules>
- <dependencies>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>3.8.2</version>
- </dependency>
- </dependencies>
- <build>
-
- </build>
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <systemProperties>
- <property>
- <name>mode</name>
- <value>local</value>
- </property>
- </systemProperties>
- </configuration>
- </plugin>
- <!-- Normally, dependency report takes time, skip it -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- <version>2.7</version>
-
- <configuration>
- <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-pmd-plugin</artifactId>
- <configuration>
- <targetJdk>1.7</targetJdk>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>emma-maven-plugin</artifactId>
- <version>1.0-alpha-3</version>
- <inherited>true</inherited>
- </plugin>
- </plugins>
- </reporting>
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-streaming/readme.md
----------------------------------------------------------------------
diff --git a/metron-streaming/readme.md b/metron-streaming/readme.md
deleted file mode 100644
index 5236f18..0000000
--- a/metron-streaming/readme.md
+++ /dev/null
@@ -1,13 +0,0 @@
-#Current Build
-
-The latest build of Metron-Streaming is 0.1BETA.
-
-We are still in the process of merging/porting additional features from our production code base into this open source release. This release will be followed by a number of additional beta releases until the port is complete. We will also work on getting additional documentation and user/developer guides to the community as soon as we can. At this time we offer no support for the beta software, but will try to respond to requests as promptly as we can.
-
-# Metron-Streaming
-
-Extensible set of Storm topologies and topology attributes for streaming, enriching, indexing, and storing telemetry in Hadoop. General information on Metron is available at https://metron.incubator.apache.org/
-
-# Documentation
-
-Please see documentation within each individual module for description and usage instructions. Sample topologies are provided under Metron_Topologies to get you started with the framework. We pre-assume knowledge of Hadoop, Storm, Kafka, and HBase.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-ui/Gruntfile.js
----------------------------------------------------------------------
diff --git a/metron-ui/Gruntfile.js b/metron-ui/Gruntfile.js
index 568611f..b443c3c 100644
--- a/metron-ui/Gruntfile.js
+++ b/metron-ui/Gruntfile.js
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
module.exports = function (grunt) {
grunt.initConfig({
// copies frontend assets from bower_components into project
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/244a5359/metron-ui/Makefile
----------------------------------------------------------------------
diff --git a/metron-ui/Makefile b/metron-ui/Makefile
index e268401..c22c51c 100644
--- a/metron-ui/Makefile
+++ b/metron-ui/Makefile
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
testcmd=./node_modules/istanbul/lib/cli.js cover \
./node_modules/mocha/bin/_mocha -- --check-leaks -R spec
@@ -26,4 +43,3 @@ seed:
clean:
rm -rf ./node_modules ./coverage
-