You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/05/16 18:07:55 UTC
[14/15] incubator-metron git commit: METRON-142 Simplify Parser
configuration (merrimanr via cestella) closes apache/incubator-metron#120
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java
new file mode 100644
index 0000000..f945dd0
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.bolt;
+
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Tuple;
+import org.apache.curator.test.TestingServer;
+import org.apache.metron.TestConstants;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.ConfigurationType;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class ConfiguredParserBoltTest extends BaseConfiguredBoltTest {
+
+ private Set<String> parserConfigurationTypes = new HashSet<>();
+ private String zookeeperUrl;
+
+ public static class StandAloneConfiguredParserBolt extends ConfiguredParserBolt {
+
+ public StandAloneConfiguredParserBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
+
+ @Override
+ public void execute(Tuple input) {
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ }
+
+ @Override
+ public void reloadCallback(String name, ConfigurationType type) {
+ configsUpdated.add(name);
+ }
+ }
+
+ @Before
+ public void setupConfiguration() throws Exception {
+ TestingServer testZkServer = new TestingServer(true);
+ this.zookeeperUrl = testZkServer.getConnectString();
+ byte[] globalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH);
+ ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig, zookeeperUrl);
+ parserConfigurationTypes.add(ConfigurationType.GLOBAL.getName());
+ Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH);
+ for (String sensorType : sensorEnrichmentConfigs.keySet()) {
+ ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensorType, sensorEnrichmentConfigs.get(sensorType), zookeeperUrl);
+ }
+ Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH);
+ for (String sensorType : sensorParserConfigs.keySet()) {
+ ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorType, sensorParserConfigs.get(sensorType), zookeeperUrl);
+ parserConfigurationTypes.add(sensorType);
+ }
+ }
+
+ @Test
+ public void test() throws Exception {
+ ParserConfigurations sampleConfigurations = new ParserConfigurations();
+ try {
+ StandAloneConfiguredParserBolt configuredBolt = new StandAloneConfiguredParserBolt(null);
+ configuredBolt.prepare(new HashMap(), topologyContext, outputCollector);
+ Assert.fail("A valid zookeeper url must be supplied");
+ } catch (RuntimeException e){}
+
+ configsUpdated = new HashSet<>();
+ sampleConfigurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
+ Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH);
+ for (String sensorType : sensorParserConfigs.keySet()) {
+ sampleConfigurations.updateSensorParserConfig(sensorType, sensorParserConfigs.get(sensorType));
+ }
+
+ StandAloneConfiguredParserBolt configuredBolt = new StandAloneConfiguredParserBolt(zookeeperUrl);
+ configuredBolt.prepare(new HashMap(), topologyContext, outputCollector);
+ waitForConfigUpdate(parserConfigurationTypes);
+ Assert.assertEquals(sampleConfigurations, configuredBolt.configurations);
+
+ configsUpdated = new HashSet<>();
+ Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig();
+ sampleGlobalConfig.put("newGlobalField", "newGlobalValue");
+ ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl);
+ waitForConfigUpdate(ConfigurationType.GLOBAL.getName());
+ Assert.assertEquals("Add global config field", sampleConfigurations.getGlobalConfig(), configuredBolt.configurations.getGlobalConfig());
+
+ configsUpdated = new HashSet<>();
+ sampleGlobalConfig.remove("newGlobalField");
+ ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl);
+ waitForConfigUpdate(ConfigurationType.GLOBAL.getName());
+ Assert.assertEquals("Remove global config field", sampleConfigurations, configuredBolt.configurations);
+
+ configsUpdated = new HashSet<>();
+ String sensorType = "testSensorConfig";
+ SensorParserConfig testSensorConfig = new SensorParserConfig();
+ testSensorConfig.setParserClassName("className");
+ testSensorConfig.setSensorTopic("sensorTopic");
+ testSensorConfig.setParserConfig(new HashMap<String, Object>() {{
+ put("configName", "configObject");
+ }});
+ sampleConfigurations.updateSensorParserConfig(sensorType, testSensorConfig);
+ ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorType, testSensorConfig, zookeeperUrl);
+ waitForConfigUpdate(sensorType);
+ Assert.assertEquals("Add new sensor config", sampleConfigurations, configuredBolt.configurations);
+ configuredBolt.cleanup();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java
index e17b877..c9a7d22 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java
@@ -69,7 +69,7 @@ public class ConfigurationManagerIntegrationTest {
zookeeperUrl = testZkServer.getConnectString();
client = ConfigurationsUtils.getClient(zookeeperUrl);
client.start();
- File sensorDir = new File(new File(TestConstants.SAMPLE_CONFIG_PATH), ConfigurationType.SENSOR.getDirectory());
+ File sensorDir = new File(new File(TestConstants.SAMPLE_CONFIG_PATH), ConfigurationType.ENRICHMENT.getDirectory());
sensors.addAll(Collections2.transform(
Arrays.asList(sensorDir.list())
,s -> Iterables.getFirst(Splitter.on('.').split(s), "null")
@@ -114,9 +114,9 @@ public class ConfigurationManagerIntegrationTest {
Assert.assertTrue("Global config does not exist", globalConfigFile.exists());
validateConfig("global", ConfigurationType.GLOBAL, new String(Files.readAllBytes(Paths.get(globalConfigFile.toURI()))));
for(String sensor : sensors) {
- File sensorFile = new File(configDir, ConfigurationType.SENSOR.getDirectory() + "/" + sensor + ".json");
+ File sensorFile = new File(configDir, ConfigurationType.ENRICHMENT.getDirectory() + "/" + sensor + ".json");
Assert.assertTrue(sensor + " config does not exist", sensorFile.exists());
- validateConfig(sensor, ConfigurationType.SENSOR, new String(Files.readAllBytes(Paths.get(sensorFile.toURI()))));
+ validateConfig(sensor, ConfigurationType.ENRICHMENT, new String(Files.readAllBytes(Paths.get(sensorFile.toURI()))));
}
}
@@ -164,6 +164,7 @@ public class ConfigurationManagerIntegrationTest {
}
}
});
+ Assert.assertEquals(true, foundGlobal.get());
Assert.assertEquals(sensorsInZookeeper, sensors);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java
index 3393b41..0645619 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java
@@ -37,8 +37,9 @@ public class ConfigurationsUtilsTest {
private TestingServer testZkServer;
private String zookeeperUrl;
private CuratorFramework client;
- private byte[] testGlobalConfig;
- private Map<String, byte[]> testSensorConfigMap;
+ private byte[] expectedGlobalConfig;
+ private Map<String, byte[]> expectedSensorParserConfigMap;
+ private Map<String, byte[]> expectedSensorEnrichmentConfigMap;
@Before
public void setup() throws Exception {
@@ -46,23 +47,31 @@ public class ConfigurationsUtilsTest {
zookeeperUrl = testZkServer.getConnectString();
client = ConfigurationsUtils.getClient(zookeeperUrl);
client.start();
- testGlobalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH);
- testSensorConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.SAMPLE_CONFIG_PATH);
+ expectedGlobalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH);
+ expectedSensorParserConfigMap = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH);
+ expectedSensorEnrichmentConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH);
}
@Test
public void test() throws Exception {
- Assert.assertTrue(testGlobalConfig.length > 0);
- ConfigurationsUtils.writeGlobalConfigToZookeeper(testGlobalConfig, zookeeperUrl);
- byte[] readGlobalConfigBytes = ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(client);
- Assert.assertTrue(Arrays.equals(testGlobalConfig, readGlobalConfigBytes));
+ Assert.assertTrue(expectedGlobalConfig.length > 0);
+ ConfigurationsUtils.writeGlobalConfigToZookeeper(expectedGlobalConfig, zookeeperUrl);
+ byte[] actualGlobalConfigBytes = ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(client);
+ Assert.assertTrue(Arrays.equals(expectedGlobalConfig, actualGlobalConfigBytes));
- Assert.assertTrue(testSensorConfigMap.size() > 0);
+ Assert.assertTrue(expectedSensorParserConfigMap.size() > 0);
String testSensorType = "yaf";
- byte[] testSensorConfigBytes = testSensorConfigMap.get(testSensorType);
- ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(testSensorType, testSensorConfigBytes, zookeeperUrl);
- byte[] readSensorConfigBytes = ConfigurationsUtils.readSensorEnrichmentConfigBytesFromZookeeper(testSensorType, client);
- Assert.assertTrue(Arrays.equals(testSensorConfigBytes, readSensorConfigBytes));
+ byte[] expectedSensorParserConfigBytes = expectedSensorParserConfigMap.get(testSensorType);
+ ConfigurationsUtils.writeSensorParserConfigToZookeeper(testSensorType, expectedSensorParserConfigBytes, zookeeperUrl);
+ byte[] actualSensorParserConfigBytes = ConfigurationsUtils.readSensorParserConfigBytesFromZookeeper(testSensorType, client);
+ Assert.assertTrue(Arrays.equals(expectedSensorParserConfigBytes, actualSensorParserConfigBytes));
+
+ Assert.assertTrue(expectedSensorEnrichmentConfigMap.size() > 0);
+ byte[] expectedSensorEnrichmentConfigBytes = expectedSensorEnrichmentConfigMap.get(testSensorType);
+ ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(testSensorType, expectedSensorEnrichmentConfigBytes, zookeeperUrl);
+ byte[] actualSensorEnrichmentConfigBytes = ConfigurationsUtils.readSensorEnrichmentConfigBytesFromZookeeper(testSensorType, client);
+ Assert.assertTrue(Arrays.equals(expectedSensorEnrichmentConfigBytes, actualSensorEnrichmentConfigBytes));
+
String name = "testConfig";
Map<String, Object> testConfig = new HashMap<>();
testConfig.put("stringField", "value");
@@ -74,7 +83,6 @@ public class ConfigurationsUtilsTest {
}
-
@After
public void tearDown() throws IOException {
client.close();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationTest.java
index fb45ccc..3031ab9 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationTest.java
@@ -55,7 +55,7 @@ public class ConfigurationTest {
GetDataBuilder getDataBuilder = mock(GetDataBuilder.class);
GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class);
- when(getDataBuilder.forPath(Constants.ZOOKEEPER_GLOBAL_ROOT)).thenReturn(mockGlobalData());
+ when(getDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot())).thenReturn(mockGlobalData());
when(curatorFramework.checkExists()).thenReturn(existsBuilder);
when(curatorFramework.getData()).thenReturn(getDataBuilder);
when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsTest.java
index 18e6ee8..4720929 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsTest.java
@@ -20,7 +20,6 @@ package org.apache.metron.common.configuration;
import junit.framework.Assert;
import nl.jqno.equalsverifier.EqualsVerifier;
import nl.jqno.equalsverifier.Warning;
-import org.apache.metron.common.configuration.Configurations;
import org.junit.Test;
import java.io.IOException;
@@ -32,7 +31,7 @@ public class ConfigurationsTest {
EqualsVerifier.forClass(Configurations.class).suppress(Warning.NONFINAL_FIELDS, Warning.NULL_FIELDS).usingGetClass().verify();
Configurations configurations = new Configurations();
try {
- configurations.updateConfig("someConfig", (byte[]) null);
+ configurations.updateGlobalConfig((byte[]) null);
Assert.fail("Updating a config with null should throw an IllegalStateException");
} catch(IllegalStateException e) {}
Assert.assertTrue(configurations.toString() != null && configurations.toString().length() > 0);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentConfigTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentConfigTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentConfigTest.java
index 418ee9b..dbf894b 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentConfigTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentConfigTest.java
@@ -32,7 +32,7 @@ public class SensorEnrichmentConfigTest {
@Test
public void test() throws IOException {
EqualsVerifier.forClass(SensorEnrichmentConfig.class).suppress(Warning.NONFINAL_FIELDS).usingGetClass().verify();
- Map<String, byte[]> testSensorConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.SAMPLE_CONFIG_PATH);
+ Map<String, byte[]> testSensorConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH);
byte[] sensorConfigBytes = testSensorConfigMap.get("yaf");
SensorEnrichmentConfig sensorEnrichmentConfig = SensorEnrichmentConfig.fromBytes(sensorConfigBytes);
Assert.assertNotNull(sensorEnrichmentConfig);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentUpdateConfigTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentUpdateConfigTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentUpdateConfigTest.java
index 009fc22..77035fb 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentUpdateConfigTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/SensorEnrichmentUpdateConfigTest.java
@@ -79,7 +79,7 @@ public class SensorEnrichmentUpdateConfigTest {
@Test
public void testThreatIntel() throws Exception {
- SensorEnrichmentConfig broSc = (SensorEnrichmentConfig) ConfigurationType.SENSOR.deserialize(sourceConfigStr);
+ SensorEnrichmentConfig broSc = (SensorEnrichmentConfig) ConfigurationType.ENRICHMENT.deserialize(sourceConfigStr);
SensorEnrichmentUpdateConfig config = JSONUtils.INSTANCE.load(threatIntelConfigStr, SensorEnrichmentUpdateConfig.class);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
index 989abfb..4aa5995 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
@@ -18,7 +18,7 @@
package org.apache.metron.elasticsearch.writer;
import backtype.storm.tuple.Tuple;
-import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.common.interfaces.BulkMessageWriter;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -51,7 +51,7 @@ public class ElasticsearchWriter implements BulkMessageWriter<JSONObject>, Seria
}
@Override
- public void init(Map stormConf, Configurations configurations) {
+ public void init(Map stormConf, EnrichmentConfigurations configurations) {
Map<String, Object> globalConfiguration = configurations.getGlobalConfig();
ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder();
builder.put("cluster.name", globalConfiguration.get("es.clustername"));
@@ -66,7 +66,7 @@ public class ElasticsearchWriter implements BulkMessageWriter<JSONObject>, Seria
}
@Override
- public void write(String sensorType, Configurations configurations, List<Tuple> tuples, List<JSONObject> messages) throws Exception {
+ public void write(String sensorType, EnrichmentConfigurations configurations, List<Tuple> tuples, List<JSONObject> messages) throws Exception {
SensorEnrichmentConfig sensorEnrichmentConfig = configurations.getSensorEnrichmentConfig(sensorType);
String indexPostfix = dateFormat.format(new Date());
BulkRequestBuilder bulkRequest = client.prepareBulk();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java
index 6931aff..f7d96a4 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java
@@ -55,7 +55,7 @@ public class ElasticsearchEnrichmentIntegrationTest extends EnrichmentIntegratio
if (elasticSearchComponent.hasIndex(index)) {
List<Map<String, Object>> docsFromDisk;
try {
- docs = elasticSearchComponent.getAllIndexedDocs(index, "yaf_doc");
+ docs = elasticSearchComponent.getAllIndexedDocs(index, testSensorType + "_doc");
docsFromDisk = readDocsFromDisk(hdfsDir);
System.out.println(docs.size() + " vs " + inputMessages.size() + " vs " + docsFromDisk.size());
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
index fdc8f36..6c9165c 100644
--- a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
@@ -19,6 +19,18 @@
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
+ <directory>${project.basedir}/src/main/config</directory>
+ <outputDirectory>/config</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0644</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
<directory>${project.basedir}/src/main/scripts</directory>
<outputDirectory>/bin</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/bro.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/bro.json b/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/bro.json
new file mode 100644
index 0000000..0eb34b3
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/bro.json
@@ -0,0 +1,20 @@
+{
+ "index": "bro",
+ "batchSize": 5,
+ "enrichment" : {
+ "fieldMap": {
+ "geo": ["ip_dst_addr", "ip_src_addr"],
+ "host": ["host"]
+ }
+ },
+ "threatIntel": {
+ "fieldMap": {
+ "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
+ },
+ "fieldToTypeMap": {
+ "ip_src_addr" : ["malicious_ip"],
+ "ip_dst_addr" : ["malicious_ip"]
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/snort.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/snort.json b/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/snort.json
new file mode 100644
index 0000000..9dfc80e
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/snort.json
@@ -0,0 +1,28 @@
+{
+ "index": "snort",
+ "batchSize": 1,
+ "enrichment" : {
+ "fieldMap":
+ {
+ "geo": ["ip_dst_addr", "ip_src_addr"],
+ "host": ["host"]
+ }
+ },
+ "threatIntel" : {
+ "fieldMap":
+ {
+ "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
+ },
+ "fieldToTypeMap":
+ {
+ "ip_src_addr" : ["malicious_ip"],
+ "ip_dst_addr" : ["malicious_ip"]
+ },
+ "triageConfig" : {
+ "riskLevelRules" : {
+ "not(IN_SUBNET(ip_dst_addr, '192.168.0.0/24'))" : 10
+ },
+ "aggregator" : "MAX"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/websphere.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/websphere.json b/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/websphere.json
new file mode 100644
index 0000000..b765808
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/websphere.json
@@ -0,0 +1,20 @@
+{
+ "index": "websphere",
+ "batchSize": 5,
+ "enrichment": {
+ "fieldMap": {
+ "geo": [
+ "ip_src_addr"
+ ],
+ "host": [
+ "ip_src_addr"
+ ]
+ },
+ "fieldToTypeMap": {
+ "ip_src_addr": [
+ "playful_classification"
+ ]
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/yaf.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/yaf.json b/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/yaf.json
new file mode 100644
index 0000000..4e67748
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/config/zookeeper/enrichments/yaf.json
@@ -0,0 +1,22 @@
+{
+ "index": "yaf",
+ "batchSize": 5,
+ "enrichment" : {
+ "fieldMap":
+ {
+ "geo": ["ip_dst_addr", "ip_src_addr"],
+ "host": ["host"]
+ }
+ },
+ "threatIntel": {
+ "fieldMap":
+ {
+ "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
+ },
+ "fieldToTypeMap":
+ {
+ "ip_src_addr" : ["malicious_ip"],
+ "ip_dst_addr" : ["malicious_ip"]
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml
index dcc507c..226b686 100644
--- a/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml
+++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml
@@ -156,16 +156,6 @@ components:
value: -2
spouts:
- - id: "testingSpout"
- className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
- parallelism: 1
- configMethods:
- - name: "withFilename"
- args:
- - "../metron-integration-test/src/main/resources/sample/data/SampleInput/YafExampleOutput"
- - name: "withRepeating"
- args:
- - true
- id: "kafkaSpout"
className: "storm.kafka.KafkaSpout"
constructorArgs:
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
index 7cfa34d..3e407c7 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
@@ -25,6 +25,7 @@ import backtype.storm.tuple.Tuple;
import org.apache.metron.common.Constants;
import org.apache.metron.common.bolt.ConfiguredBolt;
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt;
import org.apache.metron.common.utils.ErrorUtils;
import org.apache.metron.common.utils.MessageUtils;
import org.apache.metron.common.interfaces.BulkMessageWriter;
@@ -34,7 +35,7 @@ import org.slf4j.LoggerFactory;
import java.util.*;
-public class BulkMessageWriterBolt extends ConfiguredBolt {
+public class BulkMessageWriterBolt extends ConfiguredEnrichmentBolt {
private static final Logger LOG = LoggerFactory
.getLogger(BulkMessageWriterBolt.class);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
index b79d6c7..234c795 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
@@ -28,12 +28,12 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import org.apache.metron.common.Constants;
-import org.apache.metron.common.bolt.ConfiguredBolt;
-import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.enrichment.configuration.Enrichment;
+import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt;
+import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
-import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.apache.metron.common.utils.ErrorUtils;
+import org.apache.metron.enrichment.configuration.Enrichment;
+import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,7 +61,7 @@ import java.util.concurrent.TimeUnit;
**/
@SuppressWarnings({"rawtypes", "serial"})
-public class GenericEnrichmentBolt extends ConfiguredBolt {
+public class GenericEnrichmentBolt extends ConfiguredEnrichmentBolt {
private static final Logger LOG = LoggerFactory
.getLogger(GenericEnrichmentBolt.class);
@@ -114,8 +114,9 @@ public class GenericEnrichmentBolt extends ConfiguredBolt {
this.invalidateCacheOnReload= cacheInvalidationOnReload;
return this;
}
+
@Override
- public void reloadCallback(String name, Configurations.Type type) {
+ public void reloadCallback(String name, ConfigurationType type) {
if(invalidateCacheOnReload) {
if (cache != null) {
cache.invalidateAll();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
index 1964961..771097f 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
@@ -31,6 +31,7 @@ import com.google.common.collect.Sets;
import org.apache.metron.common.bolt.ConfiguredBolt;
import org.apache.metron.common.utils.ErrorUtils;
import org.json.simple.JSONObject;
+import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
@@ -39,7 +40,7 @@ import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
-public abstract class JoinBolt<V> extends ConfiguredBolt {
+public abstract class JoinBolt<V> extends ConfiguredEnrichmentBolt {
private static final Logger LOG = LoggerFactory
.getLogger(JoinBolt.class);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java
index 4ff387c..3063b7f 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/SplitBolt.java
@@ -24,12 +24,13 @@ import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
import org.apache.metron.common.bolt.ConfiguredBolt;
+import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt;
import java.util.Map;
import java.util.Set;
public abstract class SplitBolt<T> extends
- ConfiguredBolt {
+ ConfiguredEnrichmentBolt {
protected OutputCollector collector;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
index a364419..d8f0e73 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
@@ -18,7 +18,7 @@
package org.apache.metron.writer.hdfs;
import backtype.storm.tuple.Tuple;
-import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
import org.apache.metron.common.interfaces.BulkMessageWriter;
import org.apache.storm.hdfs.bolt.format.FileNameFormat;
import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
@@ -62,13 +62,13 @@ public class HdfsWriter implements BulkMessageWriter<JSONObject>, Serializable {
}
@Override
- public void init(Map stormConfig, Configurations configurations) {
+ public void init(Map stormConfig, EnrichmentConfigurations configurations) {
this.stormConfig = stormConfig;
}
@Override
- public void write( String sourceType
- , Configurations configurations
+ public void write(String sourceType
+ , EnrichmentConfigurations configurations
, List<Tuple> tuples
, List<JSONObject> messages
) throws Exception
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
index bcf4d04..61961a7 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
@@ -21,6 +21,7 @@ import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
import org.apache.metron.test.bolt.BaseEnrichmentBoltTest;
import org.apache.metron.common.configuration.Configurations;
import org.apache.metron.common.interfaces.BulkMessageWriter;
@@ -77,7 +78,7 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
/**
* {
* "field": "value",
- * "source.type": "yaf"
+ * "source.type": "test"
* }
*/
@Multiline
@@ -116,28 +117,28 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
bulkMessageWriterBolt.declareOutputFields(declarer);
verify(declarer, times(1)).declareStream(eq("error"), argThat(new FieldsMatcher("message")));
Map stormConf = new HashMap();
- doThrow(new Exception()).when(bulkMessageWriter).init(eq(stormConf), any(Configurations.class));
+ doThrow(new Exception()).when(bulkMessageWriter).init(eq(stormConf), any(EnrichmentConfigurations.class));
try {
bulkMessageWriterBolt.prepare(stormConf, topologyContext, outputCollector);
fail("A runtime exception should be thrown when bulkMessageWriter.init throws an exception");
} catch(RuntimeException e) {}
reset(bulkMessageWriter);
bulkMessageWriterBolt.prepare(stormConf, topologyContext, outputCollector);
- verify(bulkMessageWriter, times(1)).init(eq(stormConf), any(Configurations.class));
+ verify(bulkMessageWriter, times(1)).init(eq(stormConf), any(EnrichmentConfigurations.class));
tupleList = new ArrayList<>();
for(int i = 0; i < 4; i++) {
when(tuple.getValueByField("message")).thenReturn(messageList.get(i));
tupleList.add(tuple);
bulkMessageWriterBolt.execute(tuple);
- verify(bulkMessageWriter, times(0)).write(eq(sensorType), any(Configurations.class), eq(tupleList), eq(messageList));
+ verify(bulkMessageWriter, times(0)).write(eq(sensorType), any(EnrichmentConfigurations.class), eq(tupleList), eq(messageList));
}
when(tuple.getValueByField("message")).thenReturn(messageList.get(4));
tupleList.add(tuple);
bulkMessageWriterBolt.execute(tuple);
- verify(bulkMessageWriter, times(1)).write(eq(sensorType), any(Configurations.class), eq(tupleList), argThat(new MessageListMatcher(messageList)));
+ verify(bulkMessageWriter, times(1)).write(eq(sensorType), any(EnrichmentConfigurations.class), eq(tupleList), argThat(new MessageListMatcher(messageList)));
verify(outputCollector, times(5)).ack(tuple);
reset(outputCollector);
- doThrow(new Exception()).when(bulkMessageWriter).write(eq(sensorType), any(Configurations.class), Matchers.anyListOf(Tuple.class), Matchers.anyListOf(JSONObject.class));
+ doThrow(new Exception()).when(bulkMessageWriter).write(eq(sensorType), any(EnrichmentConfigurations.class), Matchers.anyListOf(Tuple.class), Matchers.anyListOf(JSONObject.class));
when(tuple.getValueByField("message")).thenReturn(messageList.get(0));
for(int i = 0; i < 5; i++) {
bulkMessageWriterBolt.execute(tuple);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBoltTest.java
index f760e5a..d7b3629 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBoltTest.java
@@ -47,7 +47,7 @@ public class EnrichmentJoinBoltTest extends BaseEnrichmentBoltTest {
* {
* "ip_src_addr": "ip1",
* "ip_dst_addr": "ip2",
- * "source.type": "yaf",
+ * "source.type": "test",
* "enrichedField": "enrichedValue"
* }
*/
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
index 5a09f79..a9a1637 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBoltTest.java
@@ -80,7 +80,7 @@ public class GenericEnrichmentBoltTest extends BaseEnrichmentBoltTest {
{
"field1": "value1",
"field2": "value2",
- "source.type": "yaf"
+ "source.type": "test"
}
*/
@Multiline
@@ -106,7 +106,7 @@ public class GenericEnrichmentBoltTest extends BaseEnrichmentBoltTest {
{
"field1.enrichedField1": "enrichedValue1",
"field2.enrichedField2": "enrichedValue2",
- "source.type": "yaf"
+ "source.type": "test"
}
*/
@Multiline
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java
index 7166b1b..fc18ad0 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/SplitBoltTest.java
@@ -22,8 +22,8 @@ import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
import junit.framework.Assert;
+import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.test.bolt.BaseEnrichmentBoltTest;
-import org.apache.metron.common.configuration.Configurations;
import org.json.simple.JSONObject;
import org.junit.Test;
@@ -35,11 +35,7 @@ import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.argThat;
import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.doCallRealMethod;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.*;
public class SplitBoltTest extends BaseEnrichmentBoltTest {
@@ -91,7 +87,7 @@ public class SplitBoltTest extends BaseEnrichmentBoltTest {
StandAloneSplitBolt splitBolt = spy(new StandAloneSplitBolt("zookeeperUrl"));
splitBolt.setCuratorFramework(client);
splitBolt.setTreeCache(cache);
- doCallRealMethod().when(splitBolt).reloadCallback(anyString(), any(Configurations.Type.class));
+ doCallRealMethod().when(splitBolt).reloadCallback(anyString(), any(ConfigurationType.class));
splitBolt.prepare(new HashMap(), topologyContext, outputCollector);
splitBolt.declareOutputFields(declarer);
verify(declarer, times(1)).declareStream(eq("message"), argThat(new FieldsMatcher("key", "message")));
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
index d3a5414..b109960 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBoltTest.java
@@ -42,7 +42,7 @@ public class ThreatIntelJoinBoltTest extends BaseEnrichmentBoltTest {
{
"field1": "value1",
"enrichedField1": "enrichedValue1",
- "source.type": "yaf"
+ "source.type": "test"
}
*/
@Multiline
@@ -52,7 +52,7 @@ public class ThreatIntelJoinBoltTest extends BaseEnrichmentBoltTest {
{
"field1": "value1",
"enrichedField1": "enrichedValue1",
- "source.type": "yaf",
+ "source.type": "test",
"threatintels.field.end.ts": "timing"
}
*/
@@ -63,7 +63,7 @@ public class ThreatIntelJoinBoltTest extends BaseEnrichmentBoltTest {
{
"field1": "value1",
"enrichedField1": "enrichedValue1",
- "source.type": "yaf",
+ "source.type": "test",
"threatintels.field": "threatIntelValue"
}
*/
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json b/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
new file mode 100644
index 0000000..d72851e
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
@@ -0,0 +1,51 @@
+{
+ "index": "yaf",
+ "batchSize": 5,
+ "enrichment": {
+ "fieldMap": {
+ "geo": [
+ "ip_src_addr",
+ "ip_dst_addr"
+ ],
+ "host": [
+ "ip_src_addr",
+ "ip_dst_addr"
+ ],
+ "hbaseEnrichment": [
+ "ip_src_addr",
+ "ip_dst_addr"
+ ]
+ }
+ ,"fieldToTypeMap": {
+ "ip_src_addr": [
+ "playful_classification"
+ ],
+ "ip_dst_addr": [
+ "playful_classification"
+ ]
+ }
+ },
+ "threatIntel": {
+ "fieldMap": {
+ "hbaseThreatIntel": [
+ "ip_src_addr",
+ "ip_dst_addr"
+ ]
+ },
+ "fieldToTypeMap": {
+ "ip_src_addr": [
+ "malicious_ip"
+ ],
+ "ip_dst_addr": [
+ "malicious_ip"
+ ]
+ },
+ "triageConfig" : {
+ "riskLevelRules" : {
+ "ip_src_addr == '10.0.2.3' or ip_dst_addr == '10.0.2.3'" : 10
+ },
+ "aggregator" : "MAX"
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/config/zookeeper/global.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/config/zookeeper/global.json b/metron-platform/metron-integration-test/src/main/config/zookeeper/global.json
new file mode 100644
index 0000000..721f70f
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/config/zookeeper/global.json
@@ -0,0 +1,10 @@
+{
+ "es.clustername": "metron",
+ "es.ip": "localhost",
+ "es.port": 9300,
+ "es.date.format": "yyyy.MM.dd.HH",
+ "solr.zookeeper": "localhost:2181",
+ "solr.collection": "metron",
+ "solr.numShards": 1,
+ "solr.replicationFactor": 1
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java
index ec04fe0..1a3247c 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java
@@ -34,12 +34,7 @@ public abstract class BaseIntegrationTest {
@Nullable
@Override
public Void apply(@Nullable KafkaWithZKComponent kafkaWithZKComponent) {
- topologyProperties.setProperty("kafka.zk", kafkaWithZKComponent.getZookeeperConnect());
- try {
- ConfigurationsUtils.uploadConfigsToZookeeper(TestConstants.SAMPLE_CONFIG_PATH, kafkaWithZKComponent.getZookeeperConnect());
- } catch (Exception e) {
- throw new IllegalStateException(e);
- }
+ topologyProperties.setProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY, kafkaWithZKComponent.getZookeeperConnect());
return null;
}
});
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
index b392448..818dae0 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
@@ -26,10 +26,12 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.metron.common.Constants;
import org.apache.metron.TestConstants;
import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
import org.apache.metron.hbase.TableProvider;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
import org.apache.metron.enrichment.converter.EnrichmentHelper;
+import org.apache.metron.integration.components.ConfigUploadComponent;
import org.apache.metron.integration.utils.TestUtils;
import org.apache.metron.test.utils.UnitTestHelper;
import org.apache.metron.integration.components.FluxTopologyComponent;
@@ -57,6 +59,7 @@ import java.util.Set;
import java.util.Stack;
public abstract class EnrichmentIntegrationTest extends BaseIntegrationTest {
+
private static final String SRC_IP = "ip_src_addr";
private static final String DST_IP = "ip_dst_addr";
private static final String MALICIOUS_IP_TYPE = "malicious_ip";
@@ -65,9 +68,10 @@ public abstract class EnrichmentIntegrationTest extends BaseIntegrationTest {
put("orientation", "north");
}};
private String fluxPath = "../metron-enrichment/src/main/flux/enrichment/test.yaml";
+ protected String testSensorType = "test";
protected String hdfsDir = "target/enrichmentIntegrationTest/hdfs";
- private String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "YafExampleParsed";
- private String sampleIndexedPath = TestConstants.SAMPLE_DATA_INDEXED_PATH + "YafIndexed";
+ private String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
+ private String sampleIndexedPath = TestConstants.SAMPLE_DATA_INDEXED_PATH + "TestIndexed";
public static class Provider implements TableProvider, Serializable {
@@ -140,7 +144,7 @@ public abstract class EnrichmentIntegrationTest extends BaseIntegrationTest {
@Test
public void test() throws Exception {
cleanHdfsDir(hdfsDir);
- final Configurations configurations = SampleUtil.getSampleConfigs();
+ final EnrichmentConfigurations configurations = SampleUtil.getSampleEnrichmentConfigs();
final String dateFormat = "yyyy.MM.dd.HH";
final List<byte[]> inputMessages = TestUtils.readSampleData(sampleParsedPath);
final String cf = "cf";
@@ -170,6 +174,11 @@ public abstract class EnrichmentIntegrationTest extends BaseIntegrationTest {
add(new KafkaWithZKComponent.Topic(Constants.ENRICHMENT_TOPIC, 1));
}});
+ ConfigUploadComponent configUploadComponent = new ConfigUploadComponent()
+ .withTopologyProperties(topologyProperties)
+ .withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH)
+ .withEnrichmentConfigsPath(TestConstants.SAMPLE_CONFIG_PATH);
+
//create MockHBaseTables
final MockHTable trackerTable = (MockHTable)MockHTable.Provider.addToCache(trackerHBaseTableName, cf);
final MockHTable threatIntelTable = (MockHTable)MockHTable.Provider.addToCache(threatIntelTableName, cf);
@@ -194,6 +203,7 @@ public abstract class EnrichmentIntegrationTest extends BaseIntegrationTest {
UnitTestHelper.verboseLogging();
ComponentRunner runner = new ComponentRunner.Builder()
.withComponent("kafka", kafkaComponent)
+ .withComponent("config", configUploadComponent)
.withComponent("search", searchComponent)
.withComponent("storm", fluxComponent)
.withMillisecondsBetweenAttempts(10000)
@@ -214,7 +224,7 @@ public abstract class EnrichmentIntegrationTest extends BaseIntegrationTest {
List<Map<String, Object>> docsFromDisk = readDocsFromDisk(hdfsDir);
Assert.assertEquals(docsFromDisk.size(), docs.size()) ;
Assert.assertEquals(new File(hdfsDir).list().length, 1);
- Assert.assertEquals(new File(hdfsDir).list()[0], "yaf");
+ Assert.assertEquals(new File(hdfsDir).list()[0], testSensorType);
validateAll(docsFromDisk);
}
finally {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
new file mode 100644
index 0000000..0a088cb
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration.components;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.integration.InMemoryComponent;
+import org.apache.metron.integration.UnableToStartException;
+
+import java.util.Properties;
+
+public class ConfigUploadComponent implements InMemoryComponent {
+
+ private Properties topologyProperties;
+ private String globalConfigPath;
+ private String parserConfigsPath;
+ private String enrichmentConfigsPath;
+
+ public ConfigUploadComponent withTopologyProperties(Properties topologyProperties) {
+ this.topologyProperties = topologyProperties;
+ return this;
+ }
+
+ public ConfigUploadComponent withGlobalConfigsPath(String globalConfigPath) {
+ this.globalConfigPath = globalConfigPath;
+ return this;
+ }
+
+ public ConfigUploadComponent withParserConfigsPath(String parserConfigsPath) {
+ this.parserConfigsPath = parserConfigsPath;
+ return this;
+ }
+ public ConfigUploadComponent withEnrichmentConfigsPath(String enrichmentConfigsPath) {
+ this.enrichmentConfigsPath = enrichmentConfigsPath;
+ return this;
+ }
+
+
+ @Override
+ public void start() throws UnableToStartException {
+ try {
+ ConfigurationsUtils.uploadConfigsToZookeeper(globalConfigPath, parserConfigsPath, enrichmentConfigsPath, topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY));
+ } catch (Exception e) {
+ throw new UnableToStartException(e.getMessage(), e);
+ }
+ }
+
+ public SensorParserConfig getSensorParserConfig(String sensorType) {
+ SensorParserConfig sensorParserConfig = new SensorParserConfig();
+ CuratorFramework client = ConfigurationsUtils.getClient(topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY));
+ client.start();
+ try {
+ sensorParserConfig = ConfigurationsUtils.readSensorParserConfigFromZookeeper(sensorType, client);
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ client.close();
+ }
+ return sensorParserConfig;
+ }
+
+ @Override
+ public void stop() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java
index 42d7567..f991fe2 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java
@@ -44,6 +44,7 @@ import java.util.*;
public class KafkaWithZKComponent implements InMemoryComponent {
+ public static final String ZOOKEEPER_PROPERTY = "kafka.zk";
public static class Topic {
public int numPartitions;
@@ -177,6 +178,7 @@ public class KafkaWithZKComponent implements InMemoryComponent {
payload.get(bytes);
messages.add(bytes);
}
+ consumer.close();
return messages;
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/SampleUtil.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/SampleUtil.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/SampleUtil.java
index ace428d..8e293c0 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/SampleUtil.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/SampleUtil.java
@@ -20,16 +20,33 @@ package org.apache.metron.integration.utils;
import org.apache.metron.TestConstants;
import org.apache.metron.common.configuration.Configurations;
import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.common.configuration.ParserConfigurations;
import java.io.IOException;
import java.util.Map;
public class SampleUtil {
- public static
- Configurations getSampleConfigs() throws IOException {
+ public static Configurations getSampleConfigs() throws IOException {
Configurations configurations = new Configurations();
configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
+ return configurations;
+ }
+
+ public static ParserConfigurations getSampleParserConfigs() throws IOException {
+ ParserConfigurations configurations = new ParserConfigurations();
+ configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
+ Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH);
+ for(String sensorType: sensorParserConfigs.keySet()) {
+ configurations.updateSensorParserConfig(sensorType, sensorParserConfigs.get(sensorType));
+ }
+ return configurations;
+ }
+
+ public static EnrichmentConfigurations getSampleEnrichmentConfigs() throws IOException {
+ EnrichmentConfigurations configurations = new EnrichmentConfigurations();
+ configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.SAMPLE_CONFIG_PATH);
for(String sensorType: sensorEnrichmentConfigs.keySet()) {
configurations.updateSensorEnrichmentConfig(sensorType, sensorEnrichmentConfigs.get(sensorType));
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/resources/sample/config/global.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/resources/sample/config/global.json b/metron-platform/metron-integration-test/src/main/resources/sample/config/global.json
deleted file mode 100644
index 721f70f..0000000
--- a/metron-platform/metron-integration-test/src/main/resources/sample/config/global.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "es.clustername": "metron",
- "es.ip": "localhost",
- "es.port": 9300,
- "es.date.format": "yyyy.MM.dd.HH",
- "solr.zookeeper": "localhost:2181",
- "solr.collection": "metron",
- "solr.numShards": 1,
- "solr.replicationFactor": 1
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/bro.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/bro.json b/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/bro.json
deleted file mode 100644
index 422200c..0000000
--- a/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/bro.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
- "index": "bro",
- "batchSize": 5,
- "enrichment" : {
- "fieldMap": {
- "geo": [
- "ip_dst_addr",
- "ip_src_addr"
- ],
- "host": [
- "host"
- ]
- }
- },
- "threatIntel" : {
- "fieldMap":
- {
- "hbaseThreatIntel": [
- "ip_dst_addr",
- "ip_src_addr"
- ]
- },
- "fieldToTypeMap": {
- "ip_dst_addr": ["malicious_ip"]
- , "ip_src_addr": ["malicious_ip"]
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/pcap.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/pcap.json b/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/pcap.json
deleted file mode 100644
index b4308ca..0000000
--- a/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/pcap.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "index": "pcap",
- "batchSize": 5,
- "enrichment": {
- "fieldMap": {
- "geo": [
- "ip_src_addr",
- "ip_dst_addr"
- ],
- "host": [
- "ip_src_addr",
- "ip_dst_addr"
- ]
- }
- },
- "threatIntel": {
- "fieldMap": {
- "ip": [
- "ip_src_addr",
- "ip_dst_addr"
- ]
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/snort.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/snort.json b/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/snort.json
deleted file mode 100644
index 4d0d366..0000000
--- a/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/snort.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
- "index": "snort",
- "batchSize": 1,
- "enrichment": {
- "fieldMap": {
- "geo": [
- "ip_dst_addr",
- "ip_src_addr"
- ],
- "host": [
- "host"
- ]
- }
- },
- "threatIntel": {
- "fieldMap": {
- "hbaseThreatIntel": [
- "ip_dst_addr",
- "ip_src_addr"
- ]
- },
- "fieldToTypeMap": {
- "ip_dst_addr": [
- "malicious_ip"
- ],
- "ip_src_addr": [
- "malicious_ip"
- ]
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/websphere.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/websphere.json b/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/websphere.json
deleted file mode 100644
index b765808..0000000
--- a/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/websphere.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "index": "websphere",
- "batchSize": 5,
- "enrichment": {
- "fieldMap": {
- "geo": [
- "ip_src_addr"
- ],
- "host": [
- "ip_src_addr"
- ]
- },
- "fieldToTypeMap": {
- "ip_src_addr": [
- "playful_classification"
- ]
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/yaf.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/yaf.json b/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/yaf.json
deleted file mode 100644
index d72851e..0000000
--- a/metron-platform/metron-integration-test/src/main/resources/sample/config/sensors/yaf.json
+++ /dev/null
@@ -1,51 +0,0 @@
-{
- "index": "yaf",
- "batchSize": 5,
- "enrichment": {
- "fieldMap": {
- "geo": [
- "ip_src_addr",
- "ip_dst_addr"
- ],
- "host": [
- "ip_src_addr",
- "ip_dst_addr"
- ],
- "hbaseEnrichment": [
- "ip_src_addr",
- "ip_dst_addr"
- ]
- }
- ,"fieldToTypeMap": {
- "ip_src_addr": [
- "playful_classification"
- ],
- "ip_dst_addr": [
- "playful_classification"
- ]
- }
- },
- "threatIntel": {
- "fieldMap": {
- "hbaseThreatIntel": [
- "ip_src_addr",
- "ip_dst_addr"
- ]
- },
- "fieldToTypeMap": {
- "ip_src_addr": [
- "malicious_ip"
- ],
- "ip_dst_addr": [
- "malicious_ip"
- ]
- },
- "triageConfig" : {
- "riskLevelRules" : {
- "ip_src_addr == '10.0.2.3' or ip_dst_addr == '10.0.2.3'" : 10
- },
- "aggregator" : "MAX"
- }
- }
-}
-