You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ma...@apache.org on 2017/04/27 22:13:41 UTC
[06/26] incubator-metron git commit: METRON-842 Add dynamic templates
for risk score fields (cestella via nickwallen) closes
apache/incubator-metron#523
METRON-842 Add dynamic templates for risk score fields (cestella via nickwallen) closes apache/incubator-metron#523
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/2ecabaa8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/2ecabaa8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/2ecabaa8
Branch: refs/heads/Metron_0.4.0
Commit: 2ecabaa8c0e7dab929726f8da48b98de623953ab
Parents: 58d5318
Author: cestella <ce...@gmail.com>
Authored: Wed Apr 19 08:55:02 2017 -0400
Committer: nickallen <ni...@apache.org>
Committed: Wed Apr 19 08:55:02 2017 -0400
----------------------------------------------------------------------
.../CURRENT/package/files/bro_index.template | 181 +++++++++++--------
.../CURRENT/package/files/snort_index.template | 171 ++++++++++--------
.../CURRENT/package/files/yaf_index.template | 181 +++++++++++--------
.../triage/ThreatTriageProcessor.java | 1 -
4 files changed, 307 insertions(+), 227 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2ecabaa8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
index 951d967..50eae30 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
@@ -6,93 +6,120 @@
"enabled": true
},
"dynamic_templates": [
- {
- "geo_location_point": {
- "match": "enrichments:geo:*:location_point",
- "match_mapping_type": "*",
- "mapping": {
- "type": "geo_point"
- }
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
}
- },
- {
- "geo_country": {
- "match": "enrichments:geo:*:country",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_city": {
- "match": "enrichments:geo:*:city",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_location_id": {
- "match": "enrichments:geo:*:locID",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_dma_code": {
- "match": "enrichments:geo:*:dmaCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_postal_code": {
- "match": "enrichments:geo:*:postalCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_latitude": {
- "match": "enrichments:geo:*:latitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
}
- },
- {
- "geo_longitude": {
- "match": "enrichments:geo:*:longitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
}
- },
- {
- "timestamps": {
- "match": "*:ts",
- "match_mapping_type": "*",
- "mapping": {
- "type": "date",
- "format": "epoch_millis"
- }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
}
}
+ },
+ {
+ "threat_triage_score": {
+ "mapping": {
+ "type": "float"
+ },
+ "match": "threat.triage.rules:*:score",
+ "match_mapping_type": "*"
+ }
+ },
+ {
+ "threat_triage_reason": {
+ "mapping": {
+ "type": "string"
+ },
+ "match": "threat.triage.rules:*:reason",
+ "match_mapping_type": "*"
+ }
+ },
+ {
+ "threat_triage_name": {
+ "mapping": {
+ "type": "string"
+ },
+ "match": "threat.triage.rules:*:name",
+ "match_mapping_type": "*"
+ }
+ }
],
"properties": {
"timestamp": {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2ecabaa8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
index bf943df..2311cf2 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
@@ -6,91 +6,118 @@
"enabled": true
},
"dynamic_templates": [
- {
- "geo_location_point": {
- "match": "enrichments:geo:*:location_point",
- "match_mapping_type": "*",
- "mapping": {
- "type": "geo_point"
- }
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
}
- },
- {
- "geo_country": {
- "match": "enrichments:geo:*:country",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_city": {
- "match": "enrichments:geo:*:city",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_location_id": {
- "match": "enrichments:geo:*:locID",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_dma_code": {
- "match": "enrichments:geo:*:dmaCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_postal_code": {
- "match": "enrichments:geo:*:postalCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_latitude": {
- "match": "enrichments:geo:*:latitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
}
- },
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
+ }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
+ }
+ }
+ },
+ {
+ "threat_triage_score": {
+ "mapping": {
+ "type": "float"
+ },
+ "match": "threat.triage.rules:*:score",
+ "match_mapping_type": "*"
+ }
+ },
{
- "geo_longitude": {
- "match": "enrichments:geo:*:longitude",
- "match_mapping_type": "*",
+ "threat_triage_reason": {
"mapping": {
- "type": "float"
- }
+ "type": "string"
+ },
+ "match": "threat.triage.rules:*:reason",
+ "match_mapping_type": "*"
}
},
{
- "timestamps": {
- "match": "*:ts",
- "match_mapping_type": "*",
+ "threat_triage_name": {
"mapping": {
- "type": "date",
- "format": "epoch_millis"
- }
+ "type": "string"
+ },
+ "match": "threat.triage.rules:*:name",
+ "match_mapping_type": "*"
}
}
],
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2ecabaa8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
index 7743afc..bd90929 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
@@ -6,93 +6,120 @@
"enabled": true
},
"dynamic_templates": [
- {
- "geo_location_point": {
- "match": "enrichments:geo:*:location_point",
- "match_mapping_type": "*",
- "mapping": {
- "type": "geo_point"
- }
+ {
+ "geo_location_point": {
+ "match": "enrichments:geo:*:location_point",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "geo_point"
}
- },
- {
- "geo_country": {
- "match": "enrichments:geo:*:country",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_country": {
+ "match": "enrichments:geo:*:country",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_city": {
- "match": "enrichments:geo:*:city",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_city": {
+ "match": "enrichments:geo:*:city",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_location_id": {
- "match": "enrichments:geo:*:locID",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_location_id": {
+ "match": "enrichments:geo:*:locID",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_dma_code": {
- "match": "enrichments:geo:*:dmaCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_dma_code": {
+ "match": "enrichments:geo:*:dmaCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_postal_code": {
- "match": "enrichments:geo:*:postalCode",
- "match_mapping_type": "*",
- "mapping": {
- "type": "string",
- "index": "not_analyzed"
- }
+ }
+ },
+ {
+ "geo_postal_code": {
+ "match": "enrichments:geo:*:postalCode",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
}
- },
- {
- "geo_latitude": {
- "match": "enrichments:geo:*:latitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
+ }
+ },
+ {
+ "geo_latitude": {
+ "match": "enrichments:geo:*:latitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
}
- },
- {
- "geo_longitude": {
- "match": "enrichments:geo:*:longitude",
- "match_mapping_type": "*",
- "mapping": {
- "type": "float"
- }
+ }
+ },
+ {
+ "geo_longitude": {
+ "match": "enrichments:geo:*:longitude",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "float"
}
- },
- {
- "timestamps": {
- "match": "*:ts",
- "match_mapping_type": "*",
- "mapping": {
- "type": "date",
- "format": "epoch_millis"
- }
+ }
+ },
+ {
+ "timestamps": {
+ "match": "*:ts",
+ "match_mapping_type": "*",
+ "mapping": {
+ "type": "date",
+ "format": "epoch_millis"
}
}
+ },
+ {
+ "threat_triage_score": {
+ "mapping": {
+ "type": "float"
+ },
+ "match": "threat.triage.rules:*:score",
+ "match_mapping_type": "*"
+ }
+ },
+ {
+ "threat_triage_reason": {
+ "mapping": {
+ "type": "string"
+ },
+ "match": "threat.triage.rules:*:reason",
+ "match_mapping_type": "*"
+ }
+ },
+ {
+ "threat_triage_name": {
+ "mapping": {
+ "type": "string"
+ },
+ "match": "threat.triage.rules:*:name",
+ "match_mapping_type": "*"
+ }
+ }
],
"properties": {
"timestamp": {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2ecabaa8/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
index 4d22081..8b09e85 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/threatintel/triage/ThreatTriageProcessor.java
@@ -96,7 +96,6 @@ public class ThreatTriageProcessor implements Function<Map, ThreatScore> {
Aggregators aggregators = threatTriageConfig.getAggregator();
List<Number> allScores = threatScore.getRuleScores().stream().map(score -> score.getRule().getScore()).collect(Collectors.toList());
Double aggregateScore = aggregators.aggregate(allScores, threatTriageConfig.getAggregationConfig());
-
// return the overall threat score
threatScore.setScore(aggregateScore);
return threatScore;