You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/01/26 15:19:04 UTC
[86/89] [abbrv] incubator-metron git commit: Merge branch 'flux' of
github.com:sirsean/incubator-metron
Merge branch 'flux' of github.com:sirsean/incubator-metron
This switches the topologies to run via Storm Flux, rather than the old
topology runner code.
This fixes apache/incubator-metron#10
Squashed commit of the following:
commit b6db53e933bbae692a03b1e952a5bc89f2a33d7f
Merge: bafa53c bfa6404
Author: Sean Schulte <si...@gmail.com>
Date: Thu Jan 14 11:54:24 2016 -0600
Merge branch 'master' of github.com:apache/incubator-metron into flux
commit bafa53c918ddd3c8a8f283dd2b32e16c3c60d9e7
Merge: 367c81e 9096e4a
Author: Sean Schulte <si...@gmail.com>
Date: Thu Jan 14 11:35:20 2016 -0600
Merge branch 'master' of github.com:apache/incubator-metron into flux
(This includes the big rename.)
commit 367c81edf7cf6d45f7949da314b6007f0663c82a
Merge: 5b3a14b 3c0c2db
Author: Sean Schulte <si...@gmail.com>
Date: Wed Jan 13 14:51:30 2016 -0600
Merge branch 'master' of github.com:apache/incubator-metron into flux
commit 5b3a14b750e5e416913c0bb13bfe2fde1675daf5
Author: Sean Schulte <si...@gmail.com>
Date: Wed Jan 13 11:17:49 2016 -0600
Removed old topology runner/builder Java code.
commit b92766d48f718510ca8e6af98db29c2e4f139e92
Author: Sean Schulte <si...@gmail.com>
Date: Wed Jan 13 11:04:57 2016 -0600
Removed old config files.
commit 0237f17958bef476735f3e87fe862a7ed219ad7b
Author: Sean Schulte <si...@gmail.com>
Date: Tue Jan 12 15:38:09 2016 -0600
Added Flux yaml for existing topologies.
commit 971a478024d00e566359758de86ae44e4f36a807
Author: Sean Schulte <si...@gmail.com>
Date: Tue Jan 12 15:34:46 2016 -0600
Added remote Bro topology.
commit ffe0369925d90f0a261f1ca902b5bd90ce4b3e10
Author: Sean Schulte <si...@gmail.com>
Date: Fri Jan 8 14:27:03 2016 -0600
Started local Bro topology with Flux.
commit 263c6fd498a1f28846892b6fd9fbcc8c52629fd5
Author: Sean Schulte <si...@gmail.com>
Date: Wed Jan 6 14:45:08 2016 -0600
Add Flux to POM.
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/a9d4eece
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/a9d4eece
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/a9d4eece
Branch: refs/heads/master
Commit: a9d4eece160bad5e8c374d7d4e4b22abed904d24
Parents: 9d5374c
Author: Sean Schulte <si...@gmail.com>
Authored: Fri Jan 22 12:10:14 2016 -0600
Committer: cstella <ce...@gmail.com>
Committed: Tue Jan 26 08:45:12 2016 -0500
----------------------------------------------------------------------
metron-streaming/Metron-Topologies/pom.xml | 5 +
metron-streaming/Metron-Topologies/readme.md | 46 +-
.../test/spouts/GenericInternalTestSpout.java | 6 +-
.../java/org/apache/metron/topology/Asa.java | 40 -
.../java/org/apache/metron/topology/Bro.java | 36 -
.../org/apache/metron/topology/FireEye.java | 21 -
.../java/org/apache/metron/topology/Ise.java | 39 -
.../org/apache/metron/topology/Lancope.java | 40 -
.../metron/topology/PaloAltoFirewall.java | 41 -
.../java/org/apache/metron/topology/Pcap.java | 41 -
.../org/apache/metron/topology/Sourcefire.java | 40 -
.../metron/topology/runner/AsaRunner.java | 94 --
.../metron/topology/runner/BroRunner.java | 89 --
.../metron/topology/runner/FireEyeRunner.java | 77 --
.../metron/topology/runner/ISERunner.java | 91 --
.../metron/topology/runner/LancopeRunner.java | 94 --
.../topology/runner/PaloAltoFirewallRunner.java | 95 --
.../metron/topology/runner/PcapRunner.java | 72 --
.../topology/runner/SourcefireRunner.java | 94 --
.../metron/topology/runner/TopologyRunner.java | 1048 ------------------
.../Metron_Configs/etc/env/config.properties | 62 ++
.../etc/env/environment_common.conf | 4 -
.../Metron_Configs/etc/env/es_connection.conf | 3 -
.../Metron_Configs/etc/env/hdfs_connection.conf | 2 -
.../etc/env/mysql_connection.conf | 4 -
.../topologies/asa/features_enabled.conf | 113 --
.../Metron_Configs/topologies/asa/local.yaml | 358 ++++++
.../Metron_Configs/topologies/asa/metrics.conf | 26 -
.../Metron_Configs/topologies/asa/remote.yaml | 372 +++++++
.../Metron_Configs/topologies/asa/topology.conf | 110 --
.../topologies/asa/topology_identifier.conf | 4 -
.../Metron_Configs/topologies/bro/alerts.xml | 24 -
.../topologies/bro/features_enabled.conf | 113 --
.../Metron_Configs/topologies/bro/local.yaml | 185 ++++
.../Metron_Configs/topologies/bro/metrics.conf | 26 -
.../Metron_Configs/topologies/bro/remote.yaml | 199 ++++
.../Metron_Configs/topologies/bro/topology.conf | 137 ---
.../topologies/bro/topology_identifier.conf | 4 -
.../topologies/environment_identifier.conf | 5 -
.../topologies/fireeye/features_enabled.conf | 113 --
.../topologies/fireeye/local.yaml | 358 ++++++
.../topologies/fireeye/metrics.conf | 26 -
.../topologies/fireeye/remote.yaml | 372 +++++++
.../topologies/fireeye/topology.conf | 110 --
.../topologies/fireeye/topology_identifier.conf | 4 -
.../Metron_Configs/topologies/ise/alerts.xml | 11 -
.../topologies/ise/features_enabled.conf | 113 --
.../Metron_Configs/topologies/ise/local.yaml | 185 ++++
.../Metron_Configs/topologies/ise/metrics.conf | 26 -
.../Metron_Configs/topologies/ise/remote.yaml | 199 ++++
.../Metron_Configs/topologies/ise/topology.conf | 101 --
.../topologies/ise/topology_identifier.conf | 4 -
.../topologies/lancope/alerts.xml | 11 -
.../topologies/lancope/features_enabled.conf | 113 --
.../topologies/lancope/local.yaml | 320 ++++++
.../topologies/lancope/metrics.conf | 26 -
.../topologies/lancope/remote.yaml | 334 ++++++
.../topologies/lancope/topology.conf | 101 --
.../topologies/lancope/topology_identifier.conf | 4 -
.../topologies/paloalto/features_enabled.conf | 113 --
.../topologies/paloalto/local.yaml | 165 +++
.../topologies/paloalto/metrics.conf | 26 -
.../topologies/paloalto/remote.yaml | 179 +++
.../topologies/paloalto/topology.conf | 113 --
.../paloalto/topology_identifier.conf | 4 -
.../topologies/pcap/features_enabled.conf | 118 --
.../Metron_Configs/topologies/pcap/local.yaml | 171 +++
.../Metron_Configs/topologies/pcap/metrics.conf | 26 -
.../Metron_Configs/topologies/pcap/remote.yaml | 185 ++++
.../topologies/pcap/topology.conf | 150 ---
.../topologies/pcap/topology_identifier.conf | 4 -
.../topologies/sourcefire/alerts.xml | 11 -
.../topologies/sourcefire/features_enabled.conf | 113 --
.../topologies/sourcefire/local.yaml | 358 ++++++
.../topologies/sourcefire/metrics.conf | 26 -
.../topologies/sourcefire/remote.yaml | 372 +++++++
.../topologies/sourcefire/topology.conf | 110 --
.../sourcefire/topology_identifier.conf | 4 -
metron-streaming/pom.xml | 3 +-
79 files changed, 4403 insertions(+), 4239 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/pom.xml b/metron-streaming/Metron-Topologies/pom.xml
index 9437fc6..524813a 100644
--- a/metron-streaming/Metron-Topologies/pom.xml
+++ b/metron-streaming/Metron-Topologies/pom.xml
@@ -106,6 +106,11 @@
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
+ <artifactId>flux-core</artifactId>
+ <version>${global_flux_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
<artifactId>storm</artifactId>
<version>${global_storm_version}</version>
<type>pom</type>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/readme.md
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/readme.md b/metron-streaming/Metron-Topologies/readme.md
index 586c166..e1c737d 100644
--- a/metron-streaming/Metron-Topologies/readme.md
+++ b/metron-streaming/Metron-Topologies/readme.md
@@ -6,42 +6,34 @@ This module provides example topologies that show how to drive Metron modules an
##Launching Topologies
+We use Storm Flux to launch topologies, which are each described in a YAML file.
```
+storm jar target/Metron-Topologies-0.6BETA.jar org.apache.storm.flux.Flux --local src/main/resources/Metron_Configs/topologies/bro/local.yaml --filter src/main/resources/Metron_Configs/etc/env/config.properties
-storm jar Metron-Topologies-0.6BETA.jar org.apache.metron.topology.Pcap
-storm jar Metron-Topologies-0.6BETA.jar org.apache.metron.topology.Sourcefire
-storm jar Metron-Topologies-0.6BETA.jar org.apache.metron.topology.Lancope
-storm jar Metron-Topologies-0.6BETA.jar org.apache.metron.topology.Ise
-
-Topology Options:
--config_path <arg> OPTIONAL ARGUMENT [/path/to/configs] Path to
-configuration folder. If not provided topology
-will initialize with default configs
--debug <arg> OPTIONAL ARGUMENT [true|false] Storm debugging
-enabled. Default value is true
--generator_spout <arg> REQUIRED ARGUMENT [true|false] Turn on test
-generator spout. Default is set to false. If
-test generator spout is turned on then kafka
-spout is turned off. Instead the generator
-spout will read telemetry from file and ingest
-it into a topology
--h Display help menue
--local_mode <arg> REQUIRED ARGUMENT [true|false] Local mode or
-cluster mode. If set to true the topology will
-run in local mode. If set to false the topology
-will be deployed to Storm nimbus
+storm jar target/Metron-Topologies-0.6BETA.jar org.apache.storm.flux.Flux --remote src/main/resources/Metron_Configs/topologies/bro/remote.yaml --filter src/main/resources/Metron_Configs/etc/env/config.properties
```
+Note that if you use `--local` it will run the topology in local mode, using test data. If you use `--remote` it will attempt to connect to and deploy to Storm Nimbus.
+
+Each topology's YAML files are responsible for either connecting to a real spout or enabling their own testing spout. This is the primary reason different `local.yaml` and `remote.yaml` files are provided for each topology.
+
##Topology Configs
The sample topologies provided use a specific directory structure. The example directory structure was checked in here:
```
-https://github.com/apache/incubator-metron-streaming/tree/master/Metron-Topologies/src/main/resources/Metron_Configs
+https://github.com/apache/incubator-metron/tree/master/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs
```
-topology.conf - settings specific to each topology
-features_enabled.conf - turn on and off features for each topology and control parallelism
-metrics.conf - export definitions for metrics to Graphite
-topology_dentifier.conf - customer-specific tag (since we deploy to multiple data centers we need to identify where the alerts are coming from and what topologies we are looking at when we need to debug)
+Each topology has a `local.yaml` and a `remote.yaml` file to support local mode and remote mode, respectively.
+
+These topology configurations have variables that can be replaced by the `--filter` option to Flux. These variables are in `src/main/resources/OpenSOC_Configs/etc/env/config.properties`, and apply to:
+
+- Kafka
+- Elasticsearch
+- MySQL
+- Metrics
+- Bolt acks/emits/fails
+- Host enrichment
+- HDFS
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java
index c5233b5..19e4f37 100644
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java
+++ b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/test/spouts/GenericInternalTestSpout.java
@@ -55,13 +55,13 @@ public class GenericInternalTestSpout extends BaseRichSpout {
_filename = filename;
return this;
}
- public GenericInternalTestSpout withMilisecondDelay(int delay)
+ public GenericInternalTestSpout withMillisecondDelay(Integer delay)
{
_delay = delay;
return this;
}
- public GenericInternalTestSpout withRepeating(boolean repeating)
+ public GenericInternalTestSpout withRepeating(Boolean repeating)
{
_repeating = repeating;
return this;
@@ -111,4 +111,4 @@ public class GenericInternalTestSpout extends BaseRichSpout {
declarer.declare(new Fields("message"));
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Asa.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Asa.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Asa.java
deleted file mode 100644
index 4163f53..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Asa.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.topology;
-
-import org.apache.commons.configuration.ConfigurationException;
-
-import backtype.storm.generated.InvalidTopologyException;
-
-import org.apache.metron.topology.runner.AsaRunner;
-import org.apache.metron.topology.runner.TopologyRunner;
-
-
-/**
- * Topology for processing Asa messages
- *
- */
-public class Asa{
-
- public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException {
-
- TopologyRunner runner = new AsaRunner();
- runner.initTopology(args, "asa");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Bro.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Bro.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Bro.java
deleted file mode 100644
index 5021f8a..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Bro.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.topology;
-
-import org.apache.commons.configuration.ConfigurationException;
-
-import backtype.storm.generated.InvalidTopologyException;
-
-import org.apache.metron.topology.runner.BroRunner;
-import org.apache.metron.topology.runner.TopologyRunner;
-
-public class Bro{
-
- public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException {
-
- TopologyRunner runner = new BroRunner();
- runner.initTopology(args, "bro");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/FireEye.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/FireEye.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/FireEye.java
deleted file mode 100644
index 9524205..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/FireEye.java
+++ /dev/null
@@ -1,21 +0,0 @@
-package org.apache.metron.topology;
-
-import org.apache.commons.configuration.ConfigurationException;
-import backtype.storm.generated.InvalidTopologyException;
-import org.apache.metron.topology.runner.FireEyeRunner;
-import org.apache.metron.topology.runner.TopologyRunner;
-
-
-/**
- * Topology for processing FireEye syslog messages
- *
- */
-public class FireEye {
-
- public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException {
-
- TopologyRunner runner = new FireEyeRunner();
- runner.initTopology(args, "fireeye");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Ise.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Ise.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Ise.java
deleted file mode 100644
index 3970e3c..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Ise.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.topology;
-
-import org.apache.commons.configuration.ConfigurationException;
-
-import backtype.storm.generated.InvalidTopologyException;
-
-import org.apache.metron.topology.runner.ISERunner;
-import org.apache.metron.topology.runner.TopologyRunner;
-
-/**
- * Topology for processing Ise messages
- *
- */
-public class Ise{
-
- public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException {
-
- TopologyRunner runner = new ISERunner();
- runner.initTopology(args, "ise");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Lancope.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Lancope.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Lancope.java
deleted file mode 100644
index 4e68487..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Lancope.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.topology;
-
-import org.apache.commons.configuration.ConfigurationException;
-
-import backtype.storm.generated.InvalidTopologyException;
-
-import org.apache.metron.topology.runner.LancopeRunner;
-import org.apache.metron.topology.runner.TopologyRunner;
-
-
-/**
- * Topology for processing Lancope messages
- *
- */
-public class Lancope{
-
- public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException {
-
- TopologyRunner runner = new LancopeRunner();
- runner.initTopology(args, "lancope");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/PaloAltoFirewall.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/PaloAltoFirewall.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/PaloAltoFirewall.java
deleted file mode 100644
index 7df0e0a..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/PaloAltoFirewall.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.topology;
-
-import org.apache.commons.configuration.ConfigurationException;
-
-import backtype.storm.generated.InvalidTopologyException;
-
-import org.apache.metron.topology.runner.AsaRunner;
-import org.apache.metron.topology.runner.PaloAltoFirewallRunner;
-import org.apache.metron.topology.runner.TopologyRunner;
-
-
-/**
- * Topology for processing Palo Alto Firewall Syslog messages
- *
- */
-public class PaloAltoFirewall {
-
- public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException {
-
- TopologyRunner runner = new PaloAltoFirewallRunner();
- runner.initTopology(args, "paloalto");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Pcap.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Pcap.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Pcap.java
deleted file mode 100644
index bae475c..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Pcap.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.topology;
-
-import org.apache.commons.configuration.ConfigurationException;
-
-import backtype.storm.generated.InvalidTopologyException;
-
-import org.apache.metron.topology.runner.PcapRunner;
-import org.apache.metron.topology.runner.TopologyRunner;
-
-
-/**
- * Topology for processing raw packet messages
- *
- */
-
-public class Pcap{
-
- public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException {
-
- TopologyRunner runner = new PcapRunner();
- runner.initTopology(args, "pcap");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Sourcefire.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Sourcefire.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Sourcefire.java
deleted file mode 100644
index da45b08..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/Sourcefire.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.topology;
-
-import org.apache.commons.configuration.ConfigurationException;
-
-import org.apache.metron.topology.runner.SourcefireRunner;
-import org.apache.metron.topology.runner.TopologyRunner;
-
-import backtype.storm.generated.InvalidTopologyException;
-
-
-/**
- * Topology for processing Sourcefire messages
- *
- */
-public class Sourcefire{
-
- public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException {
-
- TopologyRunner runner = new SourcefireRunner();
- runner.initTopology(args, "sourcefire");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/AsaRunner.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/AsaRunner.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/AsaRunner.java
deleted file mode 100644
index 138d4e3..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/AsaRunner.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.topology.runner;
-
-import org.apache.metron.filters.GenericMessageFilter;
-import org.apache.metron.parser.interfaces.MessageParser;
-import org.apache.metron.parsing.AbstractParserBolt;
-import org.apache.metron.parsing.TelemetryParserBolt;
-import org.apache.metron.test.spouts.GenericInternalTestSpout;
-
-public class AsaRunner extends TopologyRunner{
-
- static String test_file_path = "SampleInput/AsaOutput";
-
- @Override
- public boolean initializeParsingBolt(String topology_name,
- String name) {
- try {
-
- String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1);
-
- System.out.println("[Metron] ------" + name + " is initializing from " + messageUpstreamComponent);
-
-
- String class_name = config.getString("bolt.parser.adapter");
-
- if(class_name == null)
- {
- System.out.println("[Metron] Parser adapter not set. Please set bolt.indexing.adapter in topology.conf");
- throw new Exception("Parser adapter not set");
- }
-
- Class loaded_class = Class.forName(class_name);
- MessageParser parser = (MessageParser) loaded_class.newInstance();
-
-
- AbstractParserBolt parser_bolt = new TelemetryParserBolt()
- .withMessageParser(parser)
- .withOutputFieldName(topology_name)
- .withMessageFilter(new GenericMessageFilter())
- .withMetricConfig(config);
-
- builder.setBolt(name, parser_bolt,
- config.getInt("bolt.parser.parallelism.hint"))
- .shuffleGrouping(messageUpstreamComponent)
- .setNumTasks(config.getInt("bolt.parser.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
-
- return true;
- }
-
- @Override
- public boolean initializeTestingSpout(String name) {
- try {
-
- System.out.println("[Metron] Initializing Test Spout");
-
- GenericInternalTestSpout testSpout = new GenericInternalTestSpout()
- .withFilename(test_file_path).withRepeating(
- config.getBoolean("spout.test.parallelism.repeat"));
-
- builder.setSpout(name, testSpout,
- config.getInt("spout.test.parallelism.hint")).setNumTasks(
- config.getInt("spout.test.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
- return true;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/BroRunner.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/BroRunner.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/BroRunner.java
deleted file mode 100644
index ea26d0d..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/BroRunner.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.topology.runner;
-
-import org.apache.metron.filters.GenericMessageFilter;
-import org.apache.metron.parser.interfaces.MessageParser;
-import org.apache.metron.parsing.AbstractParserBolt;
-import org.apache.metron.parsing.TelemetryParserBolt;
-import org.apache.metron.test.spouts.GenericInternalTestSpout;
-
-public class BroRunner extends TopologyRunner{
-
- static String test_file_path = "SampleInput/BroExampleOutput";
-
- @Override
- public boolean initializeParsingBolt(String topology_name,
- String name) {
- try {
-
- String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1);
-
- System.out.println("[Metron] ------" + name + " is initializing from " + messageUpstreamComponent);
-
- String class_name = config.getString("bolt.parser.adapter");
-
- if(class_name == null)
- {
- System.out.println("[Metron] Parser adapter not set. Please set bolt.indexing.adapter in topology.conf");
- throw new Exception("Parser adapter not set");
- }
-
- Class loaded_class = Class.forName(class_name);
- MessageParser parser = (MessageParser) loaded_class.newInstance();
-
- AbstractParserBolt parser_bolt = new TelemetryParserBolt()
- .withMessageParser(parser)
- .withOutputFieldName(topology_name)
- .withMessageFilter(new GenericMessageFilter())
- .withMetricConfig(config);
-
- builder.setBolt(name, parser_bolt,
- config.getInt("bolt.parser.parallelism.hint"))
- .shuffleGrouping(messageUpstreamComponent)
- .setNumTasks(config.getInt("bolt.parser.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
-
- return true;
- }
-
- @Override
- public boolean initializeTestingSpout(String name) {
- try {
-
- System.out.println("[Metron] Initializing Test Spout");
-
- GenericInternalTestSpout testSpout = new GenericInternalTestSpout()
- .withFilename(test_file_path).withRepeating(
- config.getBoolean("spout.test.parallelism.repeat"));
-
- builder.setSpout(name, testSpout,
- config.getInt("spout.test.parallelism.hint")).setNumTasks(
- config.getInt("spout.test.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/FireEyeRunner.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/FireEyeRunner.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/FireEyeRunner.java
deleted file mode 100644
index d6bd29c..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/FireEyeRunner.java
+++ /dev/null
@@ -1,77 +0,0 @@
-package org.apache.metron.topology.runner;
-
-import org.apache.metron.filters.GenericMessageFilter;
-import org.apache.metron.parser.interfaces.MessageParser;
-import org.apache.metron.parsing.AbstractParserBolt;
-import org.apache.metron.parsing.TelemetryParserBolt;
-import org.apache.metron.test.spouts.GenericInternalTestSpout;
-
-public class FireEyeRunner extends TopologyRunner{
-
- static String test_file_path = "SampleInput/FireeyeExampleOutput";
-
- @Override
- public boolean initializeParsingBolt(String topology_name,
- String name) {
- try {
-
- String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1);
-
- System.out.println("[Metron] ------" + name + " is initializing from " + messageUpstreamComponent);
-
-
- String class_name = config.getString("bolt.parser.adapter");
-
- if(class_name == null)
- {
- System.out.println("[Metron] Parser adapter not set. Please set bolt.indexing.adapter in topology.conf");
- throw new Exception("Parser adapter not set");
- }
-
- Class loaded_class = Class.forName(class_name);
- MessageParser parser = (MessageParser) loaded_class.newInstance();
-
-
- AbstractParserBolt parser_bolt = new TelemetryParserBolt()
- .withMessageParser(parser)
- .withOutputFieldName(topology_name)
- .withMessageFilter(new GenericMessageFilter())
- .withMetricConfig(config);
-
- builder.setBolt(name, parser_bolt,
- config.getInt("bolt.parser.parallelism.hint"))
- .shuffleGrouping(messageUpstreamComponent)
- .setNumTasks(config.getInt("bolt.parser.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
-
- return true;
- }
-
- @Override
- public boolean initializeTestingSpout(String name) {
- try {
-
- System.out.println("[Metron] Initializing Test Spout");
-
- GenericInternalTestSpout testSpout = new GenericInternalTestSpout()
- .withFilename(test_file_path).withRepeating(
- config.getBoolean("spout.test.parallelism.repeat"));
-
- builder.setSpout(name, testSpout,
- config.getInt("spout.test.parallelism.hint")).setNumTasks(
- config.getInt("spout.test.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
- return true;
- }
-
-
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/ISERunner.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/ISERunner.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/ISERunner.java
deleted file mode 100644
index f34034d..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/ISERunner.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.topology.runner;
-
-import org.apache.metron.filters.GenericMessageFilter;
-import org.apache.metron.parser.interfaces.MessageParser;
-import org.apache.metron.parsing.AbstractParserBolt;
-import org.apache.metron.parsing.TelemetryParserBolt;
-import org.apache.metron.test.spouts.GenericInternalTestSpout;
-
-public class ISERunner extends TopologyRunner{
-
- static String test_file_path = "SampleInput/ISESampleOutput";
-
- @Override
- public boolean initializeParsingBolt(String topology_name,
- String name) {
- try {
-
- String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1);
-
- System.out.println("[Metron] ------" + name + " is initializing from " + messageUpstreamComponent);
-
-
- String class_name = config.getString("bolt.parser.adapter");
-
- if(class_name == null)
- {
- System.out.println("[Metron] Parser adapter not set. Please set bolt.indexing.adapter in topology.conf");
- throw new Exception("Parser adapter not set");
- }
-
- Class loaded_class = Class.forName(class_name);
- MessageParser parser = (MessageParser) loaded_class.newInstance();
-
-
- AbstractParserBolt parser_bolt = new TelemetryParserBolt()
- .withMessageParser(parser)
- .withOutputFieldName(topology_name)
- .withMessageFilter(new GenericMessageFilter())
- .withMetricConfig(config);
-
- builder.setBolt(name, parser_bolt,
- config.getInt("bolt.parser.parallelism.hint"))
- .shuffleGrouping(messageUpstreamComponent)
- .setNumTasks(config.getInt("bolt.parser.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
-
- return true;
- }
-
- @Override
- public boolean initializeTestingSpout(String name) {
- try {
-
- System.out.println("[Metron] Initializing Test Spout");
-
- GenericInternalTestSpout testSpout = new GenericInternalTestSpout()
- .withFilename(test_file_path).withRepeating(
- config.getBoolean("spout.test.parallelism.repeat"));
-
- builder.setSpout(name, testSpout,
- config.getInt("spout.test.parallelism.hint")).setNumTasks(
- config.getInt("spout.test.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/LancopeRunner.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/LancopeRunner.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/LancopeRunner.java
deleted file mode 100644
index 977ebb4..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/LancopeRunner.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.topology.runner;
-
-import org.apache.metron.filters.GenericMessageFilter;
-import org.apache.metron.parser.interfaces.MessageParser;
-import org.apache.metron.parsing.AbstractParserBolt;
-import org.apache.metron.parsing.TelemetryParserBolt;
-import org.apache.metron.parsing.parsers.BasicLancopeParser;
-import org.apache.metron.test.spouts.GenericInternalTestSpout;
-
-public class LancopeRunner extends TopologyRunner{
-
- static String test_file_path = "SampleInput/LancopeExampleOutput";
-
- @Override
- public boolean initializeParsingBolt(String topology_name,
- String name) {
- try {
-
- String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1);
-
- System.out.println("[Metron] ------" + name + " is initializing from " + messageUpstreamComponent);
-
-
- String class_name = config.getString("bolt.parser.adapter");
-
- if(class_name == null)
- {
- System.out.println("[Metron] Parser adapter not set. Please set bolt.indexing.adapter in topology.conf");
- throw new Exception("Parser adapter not set");
- }
-
- Class loaded_class = Class.forName(class_name);
- MessageParser parser = (MessageParser) loaded_class.newInstance();
-
- AbstractParserBolt parser_bolt = new TelemetryParserBolt()
- .withMessageParser(parser)
- .withOutputFieldName(topology_name)
- .withMessageFilter(new GenericMessageFilter())
- .withMetricConfig(config);
-
- builder.setBolt(name, parser_bolt,
- config.getInt("bolt.parser.parallelism.hint"))
- .shuffleGrouping(messageUpstreamComponent)
- .setNumTasks(config.getInt("bolt.parser.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
-
- return true;
- }
-
- @Override
- public boolean initializeTestingSpout(String name) {
- try {
-
- System.out.println("[Metron] Initializing Test Spout");
-
- GenericInternalTestSpout testSpout = new GenericInternalTestSpout()
- .withFilename(test_file_path).withRepeating(
- config.getBoolean("spout.test.parallelism.repeat"));
-
- builder.setSpout(name, testSpout,
- config.getInt("spout.test.parallelism.hint")).setNumTasks(
- config.getInt("spout.test.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
- return true;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/PaloAltoFirewallRunner.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/PaloAltoFirewallRunner.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/PaloAltoFirewallRunner.java
deleted file mode 100644
index 317dc07..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/PaloAltoFirewallRunner.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.topology.runner;
-
-import org.apache.metron.filters.GenericMessageFilter;
-import org.apache.metron.parser.interfaces.MessageParser;
-import org.apache.metron.parsing.AbstractParserBolt;
-import org.apache.metron.parsing.TelemetryParserBolt;
-import org.apache.metron.test.spouts.GenericInternalTestSpout;
-
-public class PaloAltoFirewallRunner extends TopologyRunner{
-
- static String test_file_path = "SampleInput/PaloaltoOutput";
-
- @Override
- public boolean initializeParsingBolt(String topology_name,
- String name) {
- try {
-
- String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1);
-
- System.out.println("[Metron] ------" + name + " is initializing from " + messageUpstreamComponent);
-
-
- String class_name = config.getString("bolt.parser.adapter");
-
- if(class_name == null)
- {
- System.out.println("[Metron] Parser adapter not set. Please set bolt.indexing.adapter in topology.conf");
- throw new Exception("Parser adapter not set");
- }
-
- Class loaded_class = Class.forName(class_name);
- MessageParser parser = (MessageParser) loaded_class.newInstance();
-
-
-
- AbstractParserBolt parser_bolt = new TelemetryParserBolt()
- .withMessageParser(parser)
- .withOutputFieldName(topology_name)
- .withMessageFilter(new GenericMessageFilter())
- .withMetricConfig(config);
-
- builder.setBolt(name, parser_bolt,
- config.getInt("bolt.parser.parallelism.hint"))
- .shuffleGrouping(messageUpstreamComponent)
- .setNumTasks(config.getInt("bolt.parser.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
-
- return true;
- }
-
- @Override
- public boolean initializeTestingSpout(String name) {
- try {
-
- System.out.println("[Metron] Initializing Test Spout");
-
- GenericInternalTestSpout testSpout = new GenericInternalTestSpout()
- .withFilename(test_file_path).withRepeating(
- config.getBoolean("spout.test.parallelism.repeat"));
-
- builder.setSpout(name, testSpout,
- config.getInt("spout.test.parallelism.hint")).setNumTasks(
- config.getInt("spout.test.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
- return true;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/PcapRunner.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/PcapRunner.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/PcapRunner.java
deleted file mode 100644
index 1d601cb..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/PcapRunner.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.topology.runner;
-
-import org.apache.metron.parsing.PcapParserBolt;
-import org.apache.metron.test.spouts.GenericInternalTestSpout;
-
-public class PcapRunner extends TopologyRunner{
-
- static String test_file_path = "SampleInput/PCAPExampleOutput";
-
-
- @Override
- public boolean initializeTestingSpout(String name) {
- try {
-
-
- System.out.println("[Metron] Initializing Test Spout");
-
-
- GenericInternalTestSpout testSpout = new GenericInternalTestSpout()
- .withFilename(test_file_path).withRepeating(
- config.getBoolean("spout.test.parallelism.repeat"));
-
- builder.setSpout(name, testSpout,
- config.getInt("spout.test.parallelism.hint")).setNumTasks(
- config.getInt("spout.test.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
- return true;
- }
-
- @Override
- boolean initializeParsingBolt(String topology_name, String name) {
- try {
-
- String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1);
-
- System.out.println("[Metron] ------" + name + " is initializing from " + messageUpstreamComponent);
-
- PcapParserBolt pcapParser = new PcapParserBolt().withTsPrecision(config.getString("bolt.parser.ts.precision"));
-
- builder.setBolt(name, pcapParser,
- config.getInt("bolt.parser.parallelism.hint"))
- .setNumTasks(config.getInt("bolt.parser.num.tasks"))
- .shuffleGrouping(messageUpstreamComponent);
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
-
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a9d4eece/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/SourcefireRunner.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/SourcefireRunner.java b/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/SourcefireRunner.java
deleted file mode 100644
index ab42e5f..0000000
--- a/metron-streaming/Metron-Topologies/src/main/java/org/apache/metron/topology/runner/SourcefireRunner.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.topology.runner;
-
-import org.apache.metron.filters.GenericMessageFilter;
-import org.apache.metron.parser.interfaces.MessageParser;
-import org.apache.metron.parsing.AbstractParserBolt;
-import org.apache.metron.parsing.TelemetryParserBolt;
-import org.apache.metron.test.spouts.GenericInternalTestSpout;
-
-public class SourcefireRunner extends TopologyRunner{
-
- static String test_file_path = "SampleInput/SourcefireExampleOutput";
-
- @Override
- public boolean initializeParsingBolt(String topology_name,
- String name) {
- try {
-
- String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1);
-
- System.out.println("[Metron] ------" + name + " is initializing from " + messageUpstreamComponent);
-
-
- String class_name = config.getString("bolt.parser.adapter");
-
- if(class_name == null)
- {
- System.out.println("[Metron] Parser adapter not set. Please set bolt.indexing.adapter in topology.conf");
- throw new Exception("Parser adapter not set");
- }
-
- Class loaded_class = Class.forName(class_name);
- MessageParser parser = (MessageParser) loaded_class.newInstance();
-
-
- AbstractParserBolt parser_bolt = new TelemetryParserBolt()
- .withMessageParser(parser)
- .withOutputFieldName(topology_name)
- .withMessageFilter(new GenericMessageFilter())
- .withMetricConfig(config);
-
- builder.setBolt(name, parser_bolt,
- config.getInt("bolt.parser.parallelism.hint"))
- .shuffleGrouping(messageUpstreamComponent)
- .setNumTasks(config.getInt("bolt.parser.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
-
- return true;
- }
-
- @Override
- public boolean initializeTestingSpout(String name) {
- try {
-
- System.out.println("[Metron] Initializing Test Spout");
-
- GenericInternalTestSpout testSpout = new GenericInternalTestSpout()
- .withFilename(test_file_path).withRepeating(
- config.getBoolean("spout.test.parallelism.repeat"));
-
- builder.setSpout(name, testSpout,
- config.getInt("spout.test.parallelism.hint")).setNumTasks(
- config.getInt("spout.test.num.tasks"));
-
- } catch (Exception e) {
- e.printStackTrace();
- System.exit(0);
- }
- return true;
- }
-
-
-
-}