You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2017/08/10 00:32:09 UTC
metron git commit: METRON-1026: threatintel_taxii_load.sh throws
exception closes apache/incubator-metron#643
Repository: metron
Updated Branches:
refs/heads/master 813adf2da -> 3f0e17cf9
METRON-1026: threatintel_taxii_load.sh throws exception closes apache/incubator-metron#643
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3f0e17cf
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3f0e17cf
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3f0e17cf
Branch: refs/heads/master
Commit: 3f0e17cf98d31b1143ee4d365c87f0e850df252a
Parents: 813adf2
Author: cstella <ce...@gmail.com>
Authored: Wed Aug 9 20:31:57 2017 -0400
Committer: cstella <ce...@gmail.com>
Committed: Wed Aug 9 20:31:57 2017 -0400
----------------------------------------------------------------------
.../roles/opentaxii/templates/collections.yml | 1 +
.../roles/opentaxii/templates/services.yml | 103 ++++++++++---------
.../dataloads/extractor/ExtractorDecorator.java | 4 +
.../TransformFilterExtractorDecorator.java | 2 +-
.../dataloads/nonbulk/taxii/TaxiiLoader.java | 16 ++-
.../nonbulk/taxii/TaxiiIntegrationTest.java | 5 +-
6 files changed, 76 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/3f0e17cf/metron-deployment/roles/opentaxii/templates/collections.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/opentaxii/templates/collections.yml b/metron-deployment/roles/opentaxii/templates/collections.yml
index 986c673..07a0e5a 100644
--- a/metron-deployment/roles/opentaxii/templates/collections.yml
+++ b/metron-deployment/roles/opentaxii/templates/collections.yml
@@ -15,4 +15,5 @@
# limitations under the License.
#
---
+collections:
# intentionally blank - managed by ansible
http://git-wip-us.apache.org/repos/asf/metron/blob/3f0e17cf/metron-deployment/roles/opentaxii/templates/services.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/opentaxii/templates/services.yml b/metron-deployment/roles/opentaxii/templates/services.yml
index 90c162e..61a1d42 100644
--- a/metron-deployment/roles/opentaxii/templates/services.yml
+++ b/metron-deployment/roles/opentaxii/templates/services.yml
@@ -15,54 +15,55 @@
# limitations under the License.
#
---
-#
-# discovery: used by a TAXII Client to discover available TAXII Service
-#
-- id: discovery
- type: discovery
- address: /services/discovery
- description: Discovery service for Apache Metron
- advertised_services:
- - inbox
- - collection
- - poll
- protocol_bindings:
- - urn:taxii.mitre.org:protocol:http:1.0
-
-#
-# inbox: used by a TAXII Client to push information to a TAXII Server
-#
-- id: inbox
- type: inbox
- address: /services/inbox
- description: Inbox for Apache Metron
- destination_collection_required: yes
- accept_all_content: yes
- supported_content:
- - urn:stix.mitre.org:xml:1.1.1
- authentication_required: no
- protocol_bindings:
- - urn:taxii.mitre.org:protocol:http:1.0
-
-#
-# collection_management: used by a TAXII Client to request information about
-# available data collections or request a subscription.
-#
-- id: collection
- type: collection_management
- address: /services/collection
- description: Collection management service for Apache Metron
- protocol_bindings:
- - urn:taxii.mitre.org:protocol:http:1.0
-
-#
-# poll: used by a TAXII Client to request information from a TAXII Server
-#
-- id: poll
- type: poll
- address: /services/poll
- description: Poll service for Apache Metron
- subscription_required: no
- authentication_required: no
- protocol_bindings:
- - urn:taxii.mitre.org:protocol:http:1.0
+services:
+ #
+ # discovery: used by a TAXII Client to discover available TAXII Service
+ #
+ - id: discovery
+ type: discovery
+ address: /services/discovery
+ description: Discovery service for Apache Metron
+ advertised_services:
+ - inbox
+ - collection
+ - poll
+ protocol_bindings:
+ - urn:taxii.mitre.org:protocol:http:1.0
+
+ #
+ # inbox: used by a TAXII Client to push information to a TAXII Server
+ #
+ - id: inbox
+ type: inbox
+ address: /services/inbox
+ description: Inbox for Apache Metron
+ destination_collection_required: yes
+ accept_all_content: yes
+ supported_content:
+ - urn:stix.mitre.org:xml:1.1.1
+ authentication_required: no
+ protocol_bindings:
+ - urn:taxii.mitre.org:protocol:http:1.0
+
+ #
+ # collection_management: used by a TAXII Client to request information about
+ # available data collections or request a subscription.
+ #
+ - id: collection
+ type: collection_management
+ address: /services/collection
+ description: Collection management service for Apache Metron
+ protocol_bindings:
+ - urn:taxii.mitre.org:protocol:http:1.0
+
+ #
+ # poll: used by a TAXII Client to request information from a TAXII Server
+ #
+ - id: poll
+ type: poll
+ address: /services/poll
+ description: Poll service for Apache Metron
+ subscription_required: no
+ authentication_required: no
+ protocol_bindings:
+ - urn:taxii.mitre.org:protocol:http:1.0
http://git-wip-us.apache.org/repos/asf/metron/blob/3f0e17cf/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java
index bf42760..1e1c037 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java
@@ -39,4 +39,8 @@ public class ExtractorDecorator implements Extractor {
public void initialize(Map<String, Object> config) {
decoratedExtractor.initialize(config);
}
+
+ public Extractor getUnderlyingExtractor() {
+ return decoratedExtractor;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3f0e17cf/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java
index 2aa76b7..790ea9f 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java
@@ -70,7 +70,7 @@ public class TransformFilterExtractorDecorator extends ExtractorDecorator {
}
public boolean existsIn(Map<String, Object> config) {
- return config.containsKey(key);
+ return config == null?false:config.containsKey(key);
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3f0e17cf/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
index 689a08f..de6ae8d 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
@@ -29,7 +29,9 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.log4j.PropertyConfigurator;
import org.apache.metron.dataloads.extractor.Extractor;
+import org.apache.metron.dataloads.extractor.ExtractorDecorator;
import org.apache.metron.dataloads.extractor.ExtractorHandler;
+import org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator;
import org.apache.metron.dataloads.extractor.stix.StixExtractor;
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig;
import org.apache.metron.common.utils.JSONUtils;
@@ -165,6 +167,16 @@ public class TaxiiLoader {
public static final long DEFAULT_TIME_BETWEEN_POLLS = ONE_HR_IN_MS;
+ public static boolean isStixExtractor(Extractor e) {
+ if(e instanceof StixExtractor ||
+ ( e instanceof ExtractorDecorator && ((ExtractorDecorator) e).getUnderlyingExtractor() instanceof StixExtractor)
+ ) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
public static void main(String... argv) throws Exception {
Configuration conf = HBaseConfiguration.create();
String zkQuorum = conf.get(HConstants.ZOOKEEPER_QUORUM);
@@ -185,8 +197,8 @@ public class TaxiiLoader {
}
Timer timer = new Timer();
- if(e instanceof StixExtractor) {
- StixExtractor extractor = (StixExtractor)e;
+ if(isStixExtractor(e)) {
+ Extractor extractor = e;
TaxiiConnectionConfig connectionConfig = TaxiiConnectionConfig.load(FileUtils.readFileToString(new File(TaxiiOptions.CONNECTION_CONFIG.get(cli))));
if(TaxiiOptions.BEGIN_TIME.has(cli)) {
Date d = DATE_FORMAT.parse(TaxiiOptions.BEGIN_TIME.get(cli));
http://git-wip-us.apache.org/repos/asf/metron/blob/3f0e17cf/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java
index f94a02a..a54c21b 100644
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.metron.dataloads.extractor.Extractor;
+import org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator;
import org.apache.metron.dataloads.extractor.stix.StixExtractor;
import org.apache.metron.enrichment.converter.EnrichmentConverter;
import org.apache.metron.enrichment.converter.EnrichmentKey;
@@ -94,7 +96,8 @@ public class TaxiiIntegrationTest {
final MockHBaseTableProvider provider = new MockHBaseTableProvider();
final Configuration config = HBaseConfiguration.create();
- TaxiiHandler handler = new TaxiiHandler(TaxiiConnectionConfig.load(taxiiConnectionConfig), new StixExtractor(), config ) {
+ Extractor extractor = new TransformFilterExtractorDecorator(new StixExtractor());
+ TaxiiHandler handler = new TaxiiHandler(TaxiiConnectionConfig.load(taxiiConnectionConfig), extractor, config ) {
@Override
protected synchronized HTableInterface createHTable(String tableInfo) throws IOException {
return provider.addToCache("threat_intel", "cf");