You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ot...@apache.org on 2017/08/30 15:04:53 UTC
[28/39] metron git commit: METRON-1136 Metron Extensions System and
Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/pom.xml
new file mode 100644
index 0000000..616bb24
--- /dev/null
+++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-asa-extension/pom.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parser-extensions</artifactId>
+ <version>0.4.1</version>
+ </parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parser-asa-extension</artifactId>
+ <name>metron-parser-asa-extension</name>
+ <version>0.4.1</version>
+ <packaging>pom</packaging>
+
+
+ <description>ASA Parser Extension for Metron</description>
+ <modules>
+ <module>metron-parser-asa</module>
+ <module>metron-parser-asa-bundle</module>
+ <module>metron-parser-asa-assembly</module>
+ </modules>
+</project>
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/pom.xml
new file mode 100644
index 0000000..0a83e12
--- /dev/null
+++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/pom.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parser-bro-extension</artifactId>
+ <version>0.4.1</version>
+ </parent>
+
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parser-bro-assembly</artifactId>
+ <version>0.4.1</version>
+ <packaging>pom</packaging>
+ <name>metron-parser-bro-assembly</name>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/src/main/assembly/assembly.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/src/main/assembly/assembly.xml
new file mode 100644
index 0000000..d292a2d
--- /dev/null
+++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-assembly/src/main/assembly/assembly.xml
@@ -0,0 +1,42 @@
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<assembly>
+ <id>archive</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}/../metron-parser-bro/src/main/config</directory>
+ <outputDirectory>/config</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0644</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/../metron-parser-bro-bundle/target</directory>
+ <includes>
+ <include>metron-parser-bro-bundle-${project.version}.bundle</include>
+ </includes>
+ <outputDirectory>/lib</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ </fileSet>
+ </fileSets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-bundle/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-bundle/pom.xml
new file mode 100644
index 0000000..26d8b6e
--- /dev/null
+++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro-bundle/pom.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parser-bro-extension</artifactId>
+ <version>0.4.1</version>
+ </parent>
+
+ <artifactId>metron-parser-bro-bundle</artifactId>
+ <version>0.4.1</version>
+ <name>metron-parser-bro-bundle</name>
+ <packaging>bundle</packaging>
+ <properties>
+ <maven.javadoc.skip>true</maven.javadoc.skip>
+ <source.skip>false</source.skip>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parser-bro</artifactId>
+ <version>0.4.1</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/README.md b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/README.md
new file mode 100644
index 0000000..2e8119a
--- /dev/null
+++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/README.md
@@ -0,0 +1,3 @@
+# bro Parser
+
+This is the bro parser
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/pom.xml b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/pom.xml
new file mode 100644
index 0000000..a22b52c
--- /dev/null
+++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/pom.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ --><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parser-bro-extension</artifactId>
+ <version>0.4.1</version>
+ </parent>
+ <artifactId>metron-parser-bro</artifactId>
+ <version>0.4.1</version>
+ <name>metron-parser-bro</name>
+ <packaging>jar</packaging>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parsers</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.atteo.classindex</groupId>
+ <artifactId>classindex</artifactId>
+ <version>${global_classindex_version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <!-- testing -->
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parser-extensions-testing</artifactId>
+ <version>${project.parent.version}</version>
+ <type>pom</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${global_jar_version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ <resource>
+ <directory>src/main/patterns</directory>
+ </resource>
+ <resource>
+ <directory>src/test/resources</directory>
+ </resource>
+ </resources>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/elasticsearch/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/elasticsearch/bro_index.template b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/elasticsearch/bro_index.template
new file mode 100644
index 0000000..18c5d9b
--- /dev/null
+++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/elasticsearch/bro_index.template
@@ -0,0 +1,972 @@
+{
+ "template": "bro_index*",
+ "mappings": {
+ "bro_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
+ }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ },
+ {
+ "threat_triage_score": {
+ "mapping": {
+ "type": "float"
+ },
+ "match": "threat.triage.rules:*:score",
+ "match_mapping_type": "*"
+ }
+ },
+ {
+ "threat_triage_reason": {
+ "mapping": {
+ "type": "string"
+ },
+ "match": "threat.triage.rules:*:reason",
+ "match_mapping_type": "*"
+ }
+ },
+ {
+ "threat_triage_name": {
+ "mapping": {
+ "type": "string"
+ },
+ "match": "threat.triage.rules:*:name",
+ "match_mapping_type": "*"
+ }
+ }
+ ],
+ "properties": {
+ /*
+ * WARNING
+ *
+ * Because Metron inserts many distinct bro records into a single ElasticSearch index
+ * by default, it encounters field collisions due to field name reuse across bro logs.
+ *
+ * Be careful when modifying this file to not unintentionally affect other logs.
+ * For instance, the "version" field exists in the HTTP, SSL, and SSH logs. If you
+ * were to only consider the SSH log, you would set the type to integer, but because
+ * in the SSL and HTTP logs version is a string, we must set the type to string.
+ */
+ /*
+ * Metron-specific fields
+ */
+ "source:type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ /*
+ * Widely-used Bro fields (potentially renamed during Metron ingest)
+ */
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "uid": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "ip_src_addr": {
+ "type": "ip"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "ip_dst_addr": {
+ "type": "ip"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ /*
+ * HTTP log support
+ * https://www.bro.org/sphinx/scripts/base/protocols/http/main.bro.html#type-HTTP::Info
+ *
+ * Notable Fields
+ * Field: password
+ * Notes: Field exists in the HTTP and FTP logs
+ *
+ * Field: capture_password
+ * Notes: Field exists in the HTTP and FTP logs
+ *
+ * Field: trans_depth
+ * Notes: Field exists in the HTTP and SMTP logs
+ *
+ * Field: user_agent
+ * Notes: Field exists in the HTTP and SMTP logs
+ *
+ * Field: version
+ * Notes: Field exists in the HTTP, SSL, and SSH logs
+ *
+ * Field: host
+ * Notes: Field exists in the HTTP and Software logs
+ *
+ * Field: username
+ * Notes: Field exists in the HTTP and RADIUS logs
+ */
+ "trans_depth": {
+ "type": "integer"
+ },
+ "method": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "host": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "uri": {
+ "type": "string",
+ "index": "not_analyzed",
+ "ignore_above": 8191
+ },
+ "referrer": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "version": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "user_agent": {
+ "type": "string"
+ },
+ "request_body_len": {
+ "type": "long"
+ },
+ "response_body_len": {
+ "type": "long"
+ },
+ "status_code": {
+ "type": "integer"
+ },
+ "status_msg": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "username": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "password": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "capture_password": {
+ "type": "boolean"
+ },
+ /*
+ * DNS log support
+ * https://www.bro.org/sphinx/scripts/base/protocols/dns/main.bro.html#type-DNS::Info
+ *
+ * Notable Fields
+ * Field: proto
+ * Notes: Field exists in the DNS, Conn, DPD, and Notice logs
+ *
+ * Field: trans_id
+ * Notes: Field exists in the DNS and DHCP logs
+ */
+ "proto": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "trans_id": {
+ "type": "long"
+ },
+ "query": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "qclass": {
+ "type": "integer"
+ },
+ "qclass_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "qtype": {
+ "type": "integer"
+ },
+ "qtype_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "rcode": {
+ "type": "integer"
+ },
+ "rcode_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "AA": {
+ "type": "boolean"
+ },
+ "TC": {
+ "type": "boolean"
+ },
+ "RD": {
+ "type": "boolean"
+ },
+ "RA": {
+ "type": "boolean"
+ },
+ "Z": {
+ "type": "integer"
+ },
+ "answers": {
+ "type": "string"
+ },
+ "rejected": {
+ "type": "boolean"
+ },
+ /*
+ * Conn log support
+ * https://www.bro.org/sphinx/scripts/base/protocols/conn/main.bro.html#type-Conn::Info
+ *
+ * Notable Fields
+ * Field: proto
+ * Notes: Field exists in the DNS, Conn, DPD, and Notice logs
+ *
+ * Field: duration
+ * Notes: Field exists in the Conn and Files logs
+ *
+ * Field: local_orig
+ * Notes: Field exists in the Conn and Files logs
+ */
+ "service": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "duration": {
+ "type": "float"
+ },
+ "orig_bytes": {
+ "type": "long",
+ "index": "not_analyzed"
+ },
+ "resp_bytes": {
+ "type": "long",
+ "index": "not_analyzed"
+ },
+ "conn_state": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "local_orig": {
+ "type": "boolean"
+ },
+ "local_resp": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "missed_bytes": {
+ "type": "long",
+ "index": "not_analyzed"
+ },
+ "history": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "orig_pkts": {
+ "type": "long",
+ "index": "not_analyzed"
+ },
+ "orig_ip_bytes": {
+ "type": "long",
+ "index": "not_analyzed"
+ },
+ "resp_pkts": {
+ "type": "long",
+ "index": "not_analyzed"
+ },
+ "resp_ip_bytes": {
+ "type": "long",
+ "index": "not_analyzed"
+ },
+ "tunnel_parents": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ /*
+ * DPD log support
+ * https://www.bro.org/sphinx-git/scripts/base/frameworks/dpd/main.bro.html#type-DPD::Info
+ *
+ * Notable Fields
+ * Field: proto
+ * Notes: Field exists in the DNS, Conn, DPD, and Notice logs
+ */
+ "analyzer": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "failure_reason": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ /*
+ * FTP log support
+ * https://www.bro.org/sphinx/scripts/base/protocols/ftp/info.bro.html#type-FTP::Info
+ *
+ * Notable Fields
+ * Field: password
+ * Notes: Field exists in the HTTP and FTP logs
+ *
+ * Field: capture_password
+ * Notes: Field exists in the HTTP and FTP logs
+ *
+ * Field: mime_type
+ * Notes: Field exists in the FTP and Files logs
+ *
+ * Field: fuid
+ * Notes: Field exists in the FTP and Notice logs
+ */
+ "user": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "command": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "arg": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "mime_type": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "file_size": {
+ "type": "long"
+ },
+ "reply_code": {
+ "type": "integer"
+ },
+ "reply_msg": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "data_channel:passive": {
+ "type": "boolean"
+ },
+ "data_channel:orig_h": {
+ "type": "ip"
+ },
+ "data_channel:resp_h": {
+ "type": "ip"
+ },
+ "data_channel:resp_p": {
+ "type": "integer"
+ },
+ "cwd": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "passive": {
+ "type": "boolean"
+ },
+ "fuid": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ /*
+ * Files log support
+ * https://www.bro.org/sphinx/scripts/base/frameworks/files/main.bro.html#type-Files::Info
+ *
+ * Notable Fields
+ * Field: tx_hosts
+ * Notes: Metron rewrites this to "ip_src_addr"
+ *
+ * Field: rx_hosts
+ * Notes: Metron rewrites this to "ip_dst_addr"
+ *
+ * Field: mime_type
+ * Notes: Field exists in the FTP and Files logs
+ */
+ "conn_uids": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "source": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "depth": {
+ "type": "integer"
+ },
+ "analyzers": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "filename": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "is_orig": {
+ "type": "boolean"
+ },
+ "seen_bytes": {
+ "type": "long"
+ },
+ "total_bytes": {
+ "type": "long"
+ },
+ "missing_bytes": {
+ "type": "long"
+ },
+ "overflow_bytes": {
+ "type": "long"
+ },
+ "timedout": {
+ "type": "boolean"
+ },
+ "parent_fuid": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "md5": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sha1": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sha256": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ /*
+ * Known::CertInfo log support
+ * https://www.bro.org/sphinx/scripts/policy/protocols/ssl/known-certs.bro.html#type-Known::CertsInfo
+ *
+ * Notable Fields
+ * Field: subject
+ * Notes: Field exists in the Known::CertInfo and SMTP logs
+ */
+ "port_num": {
+ "type": "integer"
+ },
+ "subject": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "issuer_subject": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "serial": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ /*
+ * SMTP log support
+ * https://www.bro.org/sphinx/scripts/base/protocols/smtp/main.bro.html#type-SMTP::Info
+ *
+ * Notable Fields
+ * Field: subject
+ * Notes: Field exists in the Known::CertInfo and SMTP logs
+ */
+ "helo": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "mailfrom": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "rcptto": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "date": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "from": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "to": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "reply_to": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "msg_id": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "in_reply_to": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "x_originating_ip": {
+ "type": "ip"
+ },
+ "first_received": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "second_received": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "last_reply": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ "path": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "tls": {
+ "type": "boolean"
+ },
+ "fuids": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "is_webmail": {
+ "type": "boolean"
+ },
+ /*
+ * SSL log support
+ * https://www.bro.org/sphinx/scripts/base/protocols/ssl/main.bro.html#type-SSL::Info
+ *
+ * Notable Fields
+ * Field: version
+ * Notes: Field exists in the HTTP, SSL, and SSH logs
+ */
+ "cipher": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "curve": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "server_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "resumed": {
+ "type": "boolean"
+ },
+ "last_alert": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "next_protocol": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "established": {
+ "type": "boolean"
+ },
+ /*
+ * Weird log support
+ * https://www.bro.org/sphinx/scripts/base/frameworks/notice/weird.bro.html#type-Weird::Info
+ */
+ "name": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "addl": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "notice": {
+ "type": "boolean"
+ },
+ "peer": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ /*
+ * Notice log support
+ * https://www.bro.org/sphinx/scripts/base/frameworks/notice/main.bro.html#type-Notice::Info
+ *
+ * Notable Fields
+ * Field: fuid
+ * Notes: Field exists in the FTP and Notice logs
+ *
+ * Field: proto
+ * Notes: Field exists in the DNS, Conn, DPD, and Notice logs
+ */
+ "file_mime_type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "file_desc": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "note": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "msg": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "sub": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "src": {
+ "type": "ip"
+ },
+ "dst": {
+ "type": "ip"
+ },
+ "p": {
+ "type": "integer",
+ "index": "not_analyzed"
+ },
+ "n": {
+ "type": "integer",
+ "index": "not_analyzed"
+ },
+ "src_peer": {
+ "type": "ip"
+ },
+ "peer_descr": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "actions": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "suppress_for": {
+ "type": "double",
+ "index": "not_analyzed"
+ },
+ "dropped": {
+ "type": "boolean"
+ },
+ /*
+ * DHCP log support
+ * https://www.bro.org/sphinx/scripts/base/protocols/dhcp/main.bro.html#type-DHCP::Info
+ *
+ * Notable Fields
+ * Field: trans_id
+ * Notes: Field exists in the DNS and DHCP logs
+ *
+ * Field: mac
+ * Notes: Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs
+ */
+ "mac": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "assigned_ip": {
+ "type": "ip"
+ },
+ "lease_time": {
+ "type": "float",
+ "index": "not_analyzed"
+ },
+ /*
+ * SSH log support
+ * https://www.bro.org/sphinx/scripts/base/protocols/ssh/main.bro.html#type-SSH::Info
+ *
+ * Notable Fields
+ * Field: version
+ * Notes: Field exists in the HTTP, SSL, and SSH logs
+ */
+ "auth_success": {
+ "type": "boolean"
+ },
+ "auth_attempts": {
+ "type": "integer",
+ "index": "not_analyzed"
+ },
+ "direction": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "client": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "server": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "cipher_alg": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "mac_alg": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "compression_alg": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "kex_alg": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "host_key_alg": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "host_key": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ /*
+ * Software log support
+ * https://www.bro.org/sphinx/scripts/base/frameworks/software/main.bro.html#type-Software::Info
+ *
+ * Notable Fields
+ * Field: host
+ * Notes: Field exists in the HTTP and Software logs
+ */
+ "host_p": {
+ "type": "integer",
+ "index": "not_analyzed"
+ },
+ "software_type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "version:major": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "version:minor": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "version:minor2": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "version:minor3": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "version:addl": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "unparsed_version": {
+ "type": "string",
+ "analyzer": "simple"
+ },
+ /*
+ * RADIUS log support
+ * https://www.bro.org/sphinx/scripts/base/protocols/radius/main.bro.html#type-RADIUS::Info
+ *
+ * Notable Fields
+ * Field: username
+ * Notes: Field exists in the HTTP and RADIUS logs
+ *
+ * Field: mac
+ * Notes: Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs
+ */
+ "remote_ip": {
+ "type": "ip"
+ },
+ "connect_info": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "result": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ /*
+ * X509 log support
+ * https://www.bro.org/sphinx/scripts/base/files/x509/main.bro.html#type-X509::Info
+ *
+ * Notable Fields
+ * Field: id
+ * Notes: In other bro records, the id field is of type conn_id, so it is
+ * expanded before being logged into 4 fields, all of which are addressed
+ * under the "Widely-used Bro fields" section of this template. In X509
+ * logs, however, id is a string to identify the certificate file id.
+ */
+ "id": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "certificate:version": {
+ "type": "integer",
+ "index": "not_analyzed"
+ },
+ "certificate:serial": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "certificate:subject": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "certificate:issuer": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "certificate:not_valid_before": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "certificate:not_valid_after": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "certificate:key_alg": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "certificate:sig_alg": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "certificate:key_type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "certificate:key_length": {
+ "type": "integer",
+ "index": "not_analyzed"
+ },
+ "certificate:exponent": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "certificate:curve": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "san:dns": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "san:uri": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "san:email": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "san:ip": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "basic_constraints:ca": {
+ "type": "boolean"
+ },
+ "basic_constraints:path_len": {
+ "type": "integer",
+ "index": "not_analyzed"
+ },
+ /*
+ * Known::DevicesInfo log support
+ * https://www.bro.org/sphinx/scripts/policy/misc/known-devices.bro.html#type-Known::DevicesInfo
+ *
+ * Notable Fields
+ * Field: mac
+ * Notes: Field exists in the DHCP, RADIUS, and Known::DevicesInfo logs
+ */
+ "dhcp_host_name": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/enrichments/bro.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/enrichments/bro.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/enrichments/bro.json
new file mode 100644
index 0000000..824c812
--- /dev/null
+++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/enrichments/bro.json
@@ -0,0 +1,17 @@
+{
+ "enrichment" : {
+ "fieldMap": {
+ "geo": ["ip_dst_addr", "ip_src_addr"],
+ "host": ["host"]
+ }
+ },
+ "threatIntel": {
+ "fieldMap": {
+ "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
+ },
+ "fieldToTypeMap": {
+ "ip_src_addr" : ["malicious_ip"],
+ "ip_dst_addr" : ["malicious_ip"]
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/indexing/bro.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/indexing/bro.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/indexing/bro.json
new file mode 100644
index 0000000..4ee131d
--- /dev/null
+++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/indexing/bro.json
@@ -0,0 +1,18 @@
+{
+ "hdfs" : {
+ "index": "bro",
+ "batchSize": 5,
+ "enabled" : true
+ },
+ "elasticsearch" : {
+ "index": "bro",
+ "batchSize": 5,
+ "enabled" : true
+ },
+ "solr" : {
+ "index": "bro",
+ "batchSize": 5,
+ "enabled" : true
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/parsers/bro.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/parsers/bro.json b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/parsers/bro.json
new file mode 100644
index 0000000..7cbd0c1
--- /dev/null
+++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/config/zookeeper/parsers/bro.json
@@ -0,0 +1,6 @@
+{
+ "parserClassName": "org.apache.metron.parsers.bro.BasicBroParser",
+ "sensorTopic": "bro",
+ "parserConfig": {
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
new file mode 100644
index 0000000..5264750
--- /dev/null
+++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.bro;
+
+import java.lang.invoke.MethodHandles;
+import java.text.DecimalFormat;
+import java.text.NumberFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.metron.common.Constants;
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("serial")
+public class BasicBroParser extends BasicParser {
+
+ protected static final Logger _LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final ThreadLocal<NumberFormat> DECIMAL_FORMAT = new ThreadLocal<NumberFormat>() {
+ @Override
+ protected NumberFormat initialValue() {
+ return new DecimalFormat("0.0#####");
+ }
+ };
+ private JSONCleaner cleaner = new JSONCleaner();
+
+ @Override
+ public void configure(Map<String, Object> parserConfig) {
+
+ }
+
+ @Override
+ public void init() {
+
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public List<JSONObject> parse(byte[] msg) {
+
+ _LOG.trace("[Metron] Starting to parse incoming message");
+
+ String rawMessage = null;
+ List<JSONObject> messages = new ArrayList<>();
+ try {
+ rawMessage = new String(msg, "UTF-8");
+ _LOG.trace("[Metron] Received message: {}", rawMessage);
+
+ JSONObject cleanedMessage = cleaner.clean(rawMessage);
+ _LOG.debug("[Metron] Cleaned message: {}", cleanedMessage);
+
+ if (cleanedMessage == null || cleanedMessage.isEmpty()) {
+ throw new Exception("Unable to clean message: " + rawMessage);
+ }
+
+ String key;
+ JSONObject payload;
+ if (cleanedMessage.containsKey("type")) {
+ key = cleanedMessage.get("type").toString();
+ payload = cleanedMessage;
+ } else {
+ key = cleanedMessage.keySet().iterator().next().toString();
+
+ if (key == null) {
+ throw new Exception("Unable to retrieve key for message: "
+ + rawMessage);
+ }
+
+ payload = (JSONObject) cleanedMessage.get(key);
+ }
+
+ if (payload == null) {
+ throw new Exception("Unable to retrieve payload for message: "
+ + rawMessage);
+ }
+
+ String originalString = key.toUpperCase() + " |";
+ for (Object k : payload.keySet()) {
+ Object raw = payload.get(k);
+ String value = raw.toString();
+ if (raw instanceof Double) {
+ value = DECIMAL_FORMAT.get().format(raw);
+ }
+ originalString += " " + k.toString() + ":" + value;
+ }
+ payload.put("original_string", originalString);
+
+ replaceKey(payload, Constants.Fields.TIMESTAMP.getName(), new String[]{ "ts" });
+
+ long timestamp = 0L;
+ if (payload.containsKey(Constants.Fields.TIMESTAMP.getName())) {
+ try {
+ Double broTimestamp = ((Number) payload.get(Constants.Fields.TIMESTAMP.getName())).doubleValue();
+ String broTimestampFormatted = DECIMAL_FORMAT.get().format(broTimestamp);
+ timestamp = convertToMillis(broTimestamp);
+ payload.put(Constants.Fields.TIMESTAMP.getName(), timestamp);
+ payload.put("bro_timestamp", broTimestampFormatted);
+ _LOG.trace("[Metron] new bro record - timestamp : {}", payload.get(Constants.Fields.TIMESTAMP.getName()));
+ } catch (NumberFormatException nfe) {
+ _LOG.error("[Metron] timestamp is invalid: {}", payload.get("timestamp"));
+ payload.put(Constants.Fields.TIMESTAMP.getName(), 0);
+ }
+ }
+
+ boolean ipSrcReplaced = replaceKey(payload, Constants.Fields.SRC_ADDR.getName(), new String[]{"source_ip", "id.orig_h"});
+ if (!ipSrcReplaced) {
+ replaceKeyArray(payload, Constants.Fields.SRC_ADDR.getName(), new String[]{ "tx_hosts" });
+ }
+
+ boolean ipDstReplaced = replaceKey(payload, Constants.Fields.DST_ADDR.getName(), new String[]{"dest_ip", "id.resp_h"});
+ if (!ipDstReplaced) {
+ replaceKeyArray(payload, Constants.Fields.DST_ADDR.getName(), new String[]{ "rx_hosts" });
+ }
+
+ replaceKey(payload, Constants.Fields.SRC_PORT.getName(), new String[]{"source_port", "id.orig_p"});
+ replaceKey(payload, Constants.Fields.DST_PORT.getName(), new String[]{"dest_port", "id.resp_p"});
+
+ payload.put(Constants.Fields.PROTOCOL.getName(), key);
+ _LOG.debug("[Metron] Returning parsed message: {}", payload);
+ messages.add(payload);
+ return messages;
+
+ } catch (Exception e) {
+ String message = "Unable to parse Message: " + rawMessage;
+ _LOG.error(message, e);
+ throw new IllegalStateException(message, e);
+ }
+
+ }
+
+ private Long convertToMillis(Double timestampSeconds) {
+ return ((Double) (timestampSeconds * 1000)).longValue();
+ }
+
+ private boolean replaceKey(JSONObject payload, String toKey, String[] fromKeys) {
+ for (String fromKey : fromKeys) {
+ if (payload.containsKey(fromKey)) {
+ Object value = payload.remove(fromKey);
+ payload.put(toKey, value);
+ _LOG.trace("[Metron] Added {} to {}", toKey, payload);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean replaceKeyArray(JSONObject payload, String toKey, String[] fromKeys) {
+ for (String fromKey : fromKeys) {
+ if (payload.containsKey(fromKey)) {
+ JSONArray value = (JSONArray) payload.remove(fromKey);
+ if (value != null && !value.isEmpty()) {
+ payload.put(toKey, value.get(0));
+ _LOG.trace("[Metron] Added {} to {}", toKey, payload);
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java
new file mode 100644
index 0000000..b3647d5
--- /dev/null
+++ b/metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/main/java/org/apache/metron/parsers/bro/JSONCleaner.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.bro;
+
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+public class JSONCleaner implements Serializable {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+
+ /**
+ * @param jsonString
+ * @return
+ * @throws ParseException
+ * Takes a json String as input and modifies the keys to remove any characters other than . _ a-z A-Z or 0-9
+ */
+ @SuppressWarnings({"unchecked","rawtypes"})
+ public JSONObject clean(String jsonString) throws ParseException
+ {
+ JSONParser parser = new JSONParser();
+
+
+ Map json = (Map) parser.parse(jsonString);
+ JSONObject output = new JSONObject();
+ Iterator iter = json.entrySet().iterator();
+
+ while(iter.hasNext()){
+ Map.Entry entry = (Map.Entry)iter.next();
+
+ String key = ((String)entry.getKey()).replaceAll("[^\\._a-zA-Z0-9]+","");
+ output.put(key, entry.getValue());
+ }
+
+ return output;
+ }
+
+
+ @SuppressWarnings({ "unchecked", "rawtypes", "unused" })
+ public static void main(String args[])
+ {
+ String jsonText = "{\"first_1\": 123, \"second\": [4, 5, 6], \"third\": 789}";
+ JSONCleaner cleaner = new JSONCleaner();
+ try {
+ //cleaner.clean(jsonText);
+ Map obj=new HashMap();
+ obj.put("name","foo");
+ obj.put("num", 100);
+ obj.put("balance", 1000.21);
+ obj.put("is_vip", true);
+ obj.put("nickname",null);
+ Map obj1 = new HashMap();
+ obj1.put("sourcefile", obj);
+
+ JSONObject json = new JSONObject(obj1);
+ System.out.println(json);
+
+
+
+ System.out.print(jsonText);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+}