You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/04/15 21:20:39 UTC
[2/2] incubator-metron git commit: METRON-102 Unit Tests for parsers
and enrichment adapters (merrimanr,
franklu1 via nickwallen) closes apache/incubator-metron#78
METRON-102 Unit Tests for parsers and enrichment adapters (merrimanr, franklu1 via nickwallen) closes apache/incubator-metron#78
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/9ca109b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/9ca109b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/9ca109b1
Branch: refs/heads/master
Commit: 9ca109b11362ae0f3a07d756d4ba3cf740faadc9
Parents: 4edffde
Author: merrimanr <me...@gmail.com>
Authored: Fri Apr 15 15:19:55 2016 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Fri Apr 15 15:19:55 2016 -0400
----------------------------------------------------------------------
.../converters/enrichment/EnrichmentHelper.java | 36 +
.../Metron-EnrichmentAdapters/pom.xml | 13 +-
.../enrichment/adapters/jdbc/JdbcAdapter.java | 4 +
.../adapters/cif/CIFHbaseAdapterTest.java | 225 -------
.../enrichment/adapters/geo/GeoAdapterTest.java | 94 +++
.../adapters/geo/GeoMysqlAdapterTest.java | 185 -----
.../host/HostFromJSONListAdapterTest.java | 83 +++
.../host/HostFromPropertiesFileAdapterTest.java | 106 +++
.../adapters/jdbc/MySqlConfigTest.java | 44 ++
.../simplehbase/SimpleHBaseAdapterTest.java | 117 ++++
.../simplehbase/SimpleHBaseConfigTest.java | 42 ++
.../threatintel/ThreatIntelAdapterTest.java | 148 ++++
.../threatintel/ThreatIntelConfigTest.java | 53 ++
.../adapters/whois/WhoisHBaseAdapterTest.java | 166 -----
.../metron/parsing/parsers/GrokParserTest.java | 114 ++++
metron-streaming/Metron-Testing/pom.xml | 5 +
.../integration/EnrichmentIntegrationTest.java | 4 +-
.../integration/util/EnrichmentHelper.java | 39 --
.../integration/util/mock/MockHTable.java | 672 -------------------
.../Metron-TestingUtilities/pom.xml | 73 ++
.../integration/util/mock/MockHTable.java | 672 +++++++++++++++++++
metron-streaming/pom.xml | 1 +
22 files changed, 1605 insertions(+), 1291 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/enrichment/EnrichmentHelper.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/enrichment/EnrichmentHelper.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/enrichment/EnrichmentHelper.java
new file mode 100644
index 0000000..a3d1b66
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/enrichment/EnrichmentHelper.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.hbase.converters.enrichment;
+
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.metron.reference.lookup.LookupKV;
+
+import java.io.IOException;
+
+public enum EnrichmentHelper {
+ INSTANCE;
+ EnrichmentConverter converter = new EnrichmentConverter();
+
+ public void load(HTableInterface table, String cf, Iterable<LookupKV<EnrichmentKey, EnrichmentValue>> results) throws IOException {
+ for(LookupKV<EnrichmentKey, EnrichmentValue> result : results) {
+ Put put = converter.toPut(cf, result.getKey(), result.getValue());
+ table.put(put);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-EnrichmentAdapters/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/pom.xml b/metron-streaming/Metron-EnrichmentAdapters/pom.xml
index 7399ade..2739c4e 100644
--- a/metron-streaming/Metron-EnrichmentAdapters/pom.xml
+++ b/metron-streaming/Metron-EnrichmentAdapters/pom.xml
@@ -113,11 +113,22 @@
<version>${global_junit_version}</version>
</dependency>
<dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${global_mockito_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>commons-validator</groupId>
<artifactId>commons-validator</artifactId>
<version>1.4.0</version>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>Metron-TestingUtilities</artifactId>
+ <version>0.1BETA</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<reporting>
<plugins>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java
index b21044f..9233059 100644
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/jdbc/JdbcAdapter.java
@@ -38,6 +38,10 @@ public abstract class JdbcAdapter implements EnrichmentAdapter<CacheKey>,
private JdbcConfig config;
private String host;
+ public void setStatement(Statement statement) {
+ this.statement = statement;
+ }
+
public JdbcAdapter withJdbcConfig(JdbcConfig config) {
this.config = config;
this.host = config.getHost();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapterTest.java
deleted file mode 100644
index 9e808ee..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapterTest.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.enrichment.adapters.cif;
-
-import java.net.InetAddress;
-import java.util.Properties;
-
-import org.apache.metron.enrichment.bolt.CacheKey;
-import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-import org.apache.metron.test.AbstractTestContext;
-import org.junit.Assert;
-
-
- /**
- * <ul>
- * <li>Title: CIFHbaseAdapterTest</li>
- * <li>Description: Test Class for CIGFHbaseAdapter</li>
- * <li>Created: Aug 7, 2014</li>
- * </ul>
- * @version $Revision: 1.1 $
- */
-public class CIFHbaseAdapterTest extends AbstractTestContext {
-
- private static CIFHbaseAdapter cifHbaseAdapter=null;
-
-
-
- /**
- * Constructs a new <code>CIFHbaseAdapterTest</code> instance.
- * @param name
- */
-
- public CIFHbaseAdapterTest(String name) {
- super(name);
-
- }
-
-
-
- /**
-
- * @throws java.lang.Exception
- */
- protected static void setUpBeforeClass() throws Exception {
- }
-
- /**
-
- * @throws java.lang.Exception
- */
- protected static void tearDownAfterClass() throws Exception {
- }
-
- /*
- * (non-Javadoc)
- * @see junit.framework.TestCase#setUp()
- */
-
- protected void setUp() throws Exception {
- super.setUp();
-
- Properties prop = super.getTestProperties();
- Assert.assertNotNull(prop);
-
- if(skipTests(this.getMode())){
- return;//skip tests
- }
-
- String[] zk = prop.get("kafka.zk.list").toString().split(",");
-
- for(String z : zk)
- {
- InetAddress address = InetAddress.getByName(z);
- boolean reachable = address.isReachable(100);
-
- if(!reachable)
- {
- this.setMode("local");
- //throw new Exception("Unable to reach zookeeper, skipping CIF adapter test");
- break;
- }
-
- }
-
- if(skipTests(this.getMode()))
- return;//skip tests
-
- System.out.println("kafka.zk.list ="+(String) prop.get("kafka.zk.list"));
- System.out.println("kafka.zk.list ="+(String) prop.get("kafka.zk.port"));
- System.out.println("kafka.zk.list ="+(String) prop.get("bolt.enrichment.cif.tablename"));
- if(skipTests(this.getMode())){
- System.out.println("Local Mode Skipping tests !! ");
- }else{
- cifHbaseAdapter=new CIFHbaseAdapter((String) prop.get("kafka.zk.list"), (String) prop.get("kafka.zk.port"),(String) prop.get("bolt.enrichment.cif.tablename"));
- }
- }
-
- /*
- * (non-Javadoc)
- * @see junit.framework.TestCase#tearDown()
- */
-
- protected void tearDown() throws Exception {
- super.tearDown();
- cifHbaseAdapter=null;
- }
-
- /**
- * Test method for {@link EnrichmentAdapter#initializeAdapter()}.
- */
- public void testInitializeAdapter() {
- if(skipTests(this.getMode())){
- return;//skip tests
- }else{
- Assert.assertTrue(cifHbaseAdapter.initializeAdapter());
- }
- }
-
- /**
- * Test method for {@link org.apache.metron.enrichment.adapters.cif.CIFHbaseAdapter#enrichByIP(java.lang.String)}.
- */
- public void testEnrichByIP() {
- if(skipTests(this.getMode())){
- return;//skip tests
- }else{
- Assert.assertNull(cifHbaseAdapter.enrichByIP("11.1.1"));
- }
- }
-
- /**
- * Test method for {@link org.apache.metron.enrichment.adapters.cif.CIFHbaseAdapter#enrichByDomain(java.lang.String)}.
- */
- public void testEnrichByDomain() {
- if(skipTests(this.getMode())){
- return;//skip tests
- }else{
- Assert.assertNull(cifHbaseAdapter.enrichByIP("invaliddomain"));
- }
- }
-
- /**
- * Test method for {@link org.apache.metron.enrichment.adapters.cif.CIFHbaseAdapter#enrichByEmail(java.lang.String)}.
- */
- public void testEnrichByEmail() {
- if(skipTests(this.getMode())){
- return;//skip tests
- }else{
- Assert.assertNull(cifHbaseAdapter.enrichByIP("sample@invalid.com"));
- }
- }
-
- /**
- * Test method for {@link org.apache.metron.enrichment.adapters.cif.CIFHbaseAdapter#CIFHbaseAdapter(java.lang.String, java.lang.String, java.lang.String)}.
- */
- public void testCIFHbaseAdapter() {
- if(skipTests(this.getMode())){
- return;//skip tests
- }else{
- Assert.assertNotNull(cifHbaseAdapter);
- }
- }
-
- /**
- * Test method for {@link org.apache.metron.enrichment.adapters.cif.CIFHbaseAdapter#enrich(CacheKey)}.
- */
- public void testEnrich() {
- if(skipTests(this.getMode())){
- return;//skip tests
- }else{
- cifHbaseAdapter.initializeAdapter();
- Assert.assertNotNull(cifHbaseAdapter.enrich(new CacheKey("cif", "testinvalid.metadata", null)));
-
- Assert.assertNotNull(cifHbaseAdapter.enrich(new CacheKey("cif", "ivalid.ip", null)));
- Assert.assertNotNull(cifHbaseAdapter.enrich(new CacheKey("cif", "1.1.1.10", null)));
- }
- }
-
-
- /**
- * Test method for {@link org.apache.metron.enrichment.adapters.cif.CIFHbaseAdapter#getCIFObject(java.lang.String)}.
- */
- public void testGetCIFObject() {
- if(skipTests(this.getMode())){
- return;//skip tests
- }else{
- cifHbaseAdapter.initializeAdapter();
- Assert.assertNotNull(cifHbaseAdapter.getCIFObject("testkey"));
- }
- }
- /**
- * Returns the cifHbaseAdapter.
- * @return the cifHbaseAdapter.
- */
-
- public static CIFHbaseAdapter getCifHbaseAdapter() {
- return CIFHbaseAdapterTest.cifHbaseAdapter;
- }
-
- /**
- * Sets the cifHbaseAdapter.
- * @param cifHbaseAdapter the cifHbaseAdapter.
- */
-
- public static void setCifHbaseAdapter(CIFHbaseAdapter cifHbaseAdapter) {
-
- CIFHbaseAdapterTest.cifHbaseAdapter = cifHbaseAdapter;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoAdapterTest.java
new file mode 100644
index 0000000..ec90c49
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoAdapterTest.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.geo;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.enrichment.bolt.CacheKey;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.mockito.Mockito.when;
+
+public class GeoAdapterTest {
+
+
+ private String ip = "72.163.4.161";
+
+
+ /**
+ * {
+ * "locID":"1",
+ * "country":"test country",
+ * "city":"test city",
+ * "postalCode":"test zip",
+ * "latitude":"test latitude",
+ * "longitude":"test longitude",
+ * "dmaCode":"test dma",
+ * "location_point":"test longitude,test latitude"
+ * }
+ */
+ @Multiline
+ private String expectedMessageString;
+
+ private JSONObject expectedMessage;
+
+ @Mock
+ Statement statetment;
+ @Mock
+ ResultSet resultSet, resultSet1;
+
+
+ @Before
+ public void setup() throws Exception {
+ JSONParser jsonParser = new JSONParser();
+ expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString);
+ MockitoAnnotations.initMocks(this);
+ when(statetment.executeQuery("select IPTOLOCID(\"CacheKey{field='dummy', value='72.163.4.161'}\") as ANS")).thenReturn(resultSet);
+ when(statetment.executeQuery("select * from location where locID = 1")).thenReturn(resultSet1);
+ when(resultSet.next()).thenReturn(Boolean.TRUE, Boolean.FALSE);
+ when(resultSet.getString("ANS")).thenReturn("1");
+ when(resultSet1.next()).thenReturn(Boolean.TRUE, Boolean.FALSE);
+ when(resultSet1.getString("locID")).thenReturn("1");
+ when(resultSet1.getString("country")).thenReturn("test country");
+ when(resultSet1.getString("city")).thenReturn("test city");
+ when(resultSet1.getString("postalCode")).thenReturn("test zip");
+ when(resultSet1.getString("latitude")).thenReturn("test latitude");
+ when(resultSet1.getString("longitude")).thenReturn("test longitude");
+ when(resultSet1.getString("dmaCode")).thenReturn("test dma");
+ }
+
+
+ @Test
+ public void testEnrich() throws Exception {
+ GeoAdapter geo = new GeoAdapter();
+ geo.setStatement(statetment);
+ JSONObject actualMessage = geo.enrich(new CacheKey("dummy", ip, null));
+ Assert.assertNotNull(actualMessage.get("locID"));
+ Assert.assertEquals(expectedMessage, actualMessage);
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoMysqlAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoMysqlAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoMysqlAdapterTest.java
deleted file mode 100644
index cc3fb6f..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoMysqlAdapterTest.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.enrichment.adapters.geo;
-
-import java.net.URL;
-import java.util.Properties;
-
-import org.apache.metron.enrichment.adapters.jdbc.JdbcAdapter;
-import org.apache.metron.enrichment.adapters.jdbc.MySqlConfig;
-import org.apache.metron.enrichment.bolt.CacheKey;
-import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-import org.json.simple.JSONObject;
-
-import org.apache.metron.test.AbstractSchemaTest;
-import org.junit.Assert;
-
-
-/**
- * <ul>
- * <li>Title: GeoMySqlAdapterTest</li>
- * <li>Description: Tests for GeoMySqlAdapter</li>
- * <li>Created: Aug 25, 2014</li>
- * </ul>
- * @version $Revision: 1.1 $
- */
-public class GeoMysqlAdapterTest extends AbstractSchemaTest {
-
- private static JdbcAdapter geoMySqlAdapter=null;
- private static boolean connected=false;
-
- /**
- * Constructs a new <code>GeoMysqlAdapterTest</code> instance.
- * @param name
- */
-
- public GeoMysqlAdapterTest(String name) {
- super(name);
- }
-
- /**
-
- * @throws java.lang.Exception
- */
- protected static void setUpBeforeClass() throws Exception {
- }
-
- /**
-
- * @throws java.lang.Exception
- */
- protected static void tearDownAfterClass() throws Exception {
- }
-
- /*
- * (non-Javadoc)
- * @see junit.framework.TestCase#setUp()
- */
-
- protected void setUp() throws Exception {
- super.setUp();
- Properties prop = super.getTestProperties();
- assertNotNull(prop);
- System.out.println("username ="+(String)prop.get("mysql.username"));
- if(skipTests(this.getMode())){
- System.out.println(getClass().getName()+" Skipping Tests !!Local Mode");
- return;//skip tests
- }else{
- MySqlConfig mySqlConfig = new MySqlConfig();
- mySqlConfig.setHost((String)prop.get("mysql.ip"));
- mySqlConfig.setPort(new Integer((String) prop.get("mysql.port")));
- mySqlConfig.setUsername((String)prop.get("mysql.username"));
- mySqlConfig.setPassword((String)prop.get("mysql.password"));
- mySqlConfig.setTable((String)prop.get("bolt.enrichment.geo.adapter.table"));
- JdbcAdapter geoAdapter = new GeoAdapter().withJdbcConfig(mySqlConfig);
- GeoMysqlAdapterTest.setGeoMySqlAdapter(geoAdapter);
- connected =geoMySqlAdapter.initializeAdapter();
- Assert.assertTrue(connected);
- URL schema_url = getClass().getClassLoader().getResource(
- "TestSchemas/GeoMySqlSchema.json");
- super.setSchemaJsonString(super.readSchemaFromFile(schema_url));
- }
- }
-
- /*
- * (non-Javadoc)
- * @see junit.framework.TestCase#tearDown()
- */
-
- protected void tearDown() throws Exception {
- super.tearDown();
- GeoMysqlAdapterTest.setGeoMySqlAdapter(null);
- }
-
- /**
- * Test method for {@link org.apache.metron.enrichment.adapters.geo.GeoAdapter#enrich(java.lang.String)}.
- */
- public void testEnrich() {
- if(skipTests(this.getMode())){
- return;//skip tests
- }else{
-
- try {
- JSONObject json = geoMySqlAdapter.enrich(new CacheKey("dummy", "72.163.4.161", null));
-
- //assert Geo Response is not null
- System.out.println("json ="+json);
- assertNotNull(json);
-
- Assert.assertEquals(true, super.validateJsonData(super.getSchemaJsonString(), json.toString()));
- //assert LocId is not null
- assertNotNull(json.get("locID"));
-
- //assert right LocId is being returned
- Assert.assertEquals("4522",json.get("locID"));
- } catch (Exception e) {
- e.printStackTrace();
- Assert.assertTrue(false);
- }
- }
- }
-
- /**
- * Test method for {@link EnrichmentAdapter#initializeAdapter()}.
- */
- public void testInitializeAdapter() {
- if(skipTests(this.getMode())){
- return;//skip tests
- }else{
- boolean connected =geoMySqlAdapter.initializeAdapter();
- Assert.assertTrue(connected);
- }
- }
-
- /**
- * Test method for
- *
- *
- *
- *
- *
- * {@link org.apache.metron.enrichment.adapters.geo.GeoAdapter}.
- */
- public void testGeoMysqlAdapter() {
- if(skipTests(this.getMode())){
- return;//skip tests
- }else{
- Assert.assertTrue(connected);
- }
- }
-
- /**
- * Returns the geoMySqlAdapter.
- * @return the geoMySqlAdapter.
- */
-
- public static JdbcAdapter getGeoMySqlAdapter() {
- return geoMySqlAdapter;
- }
-
- /**
- * Sets the geoMySqlAdapter.
- * @param geoMySqlAdapter the geoMySqlAdapter.
- */
-
- public static void setGeoMySqlAdapter(JdbcAdapter geoMySqlAdapter) {
-
- GeoMysqlAdapterTest.geoMySqlAdapter = geoMySqlAdapter;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapterTest.java
new file mode 100644
index 0000000..c448f1e
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromJSONListAdapterTest.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.host;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.enrichment.bolt.CacheKey;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class HostFromJSONListAdapterTest {
+
+
+ /**
+ * [
+ * {"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},
+ * {"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},
+ * {"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"},
+ * {"ip":"10.0.2.15", "local":"YES", "type":"printer", "asset_value" : "important"}
+ * ]
+ */
+ @Multiline
+ private String expectedKnownHostsString;
+
+ /**
+ * {
+ * "known_info.local":"YES",
+ * "known_info.type":"printer",
+ * "known_info.asset_value" : "important"
+ * }
+ */
+ @Multiline
+ private String expectedMessageString;
+
+ private JSONObject expectedMessage;
+ private String ip = "10.0.2.15";
+ private String ip1 = "10.0.22.22";
+
+
+ @Before
+ public void parseJSON() throws ParseException {
+ JSONParser jsonParser = new JSONParser();
+ expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString);
+ }
+
+ @Test
+ public void testEnrich() throws Exception {
+ HostFromJSONListAdapter hja = new HostFromJSONListAdapter(expectedKnownHostsString);
+ JSONObject actualMessage = hja.enrich(new CacheKey("dummy", ip, null));
+ Assert.assertNotNull(actualMessage);
+ Assert.assertEquals(expectedMessage, actualMessage);
+ actualMessage = hja.enrich(new CacheKey("dummy", ip1, null));
+ JSONObject emptyJson = new JSONObject();
+ Assert.assertEquals(emptyJson, actualMessage);
+ }
+
+
+ @Test
+ public void testInitializeAdapter() throws Exception {
+ HostFromJSONListAdapter hja = new HostFromJSONListAdapter(expectedKnownHostsString);
+ Assert.assertTrue(hja.initializeAdapter());
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapterTest.java
new file mode 100644
index 0000000..ea5cabe
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapterTest.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.host;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.enrichment.bolt.CacheKey;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+
+public class HostFromPropertiesFileAdapterTest {
+
+ /**
+ * [
+ * {"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},
+ * {"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},
+ * {"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"},
+ * {"ip":"10.0.2.15", "local":"YES", "type":"printer", "asset_value" : "important"}
+ * ]
+ */
+ @Multiline
+ private String expectedKnownHostsString;
+
+ /**
+ * {
+ * "known_info":
+ * {"asset_value":"important",
+ * "type":"printer","local":"YES"
+ * }
+ * }
+ */
+ @Multiline
+ private String expectedMessageString;
+
+ private JSONObject expectedMessage;
+ private String ip = "10.0.2.15";
+ private String ip1 = "10.0.22.22";
+
+ @Before
+ public void parseJSON() throws ParseException {
+ JSONParser jsonParser = new JSONParser();
+ expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString);
+ }
+
+ @Test
+ public void testEnrich() throws Exception {
+ Map<String, JSONObject> mapKnownHosts = new HashMap<>();
+ JSONArray jsonArray = (JSONArray) JSONValue.parse(expectedKnownHostsString);
+ Iterator jsonArrayIterator = jsonArray.iterator();
+ while(jsonArrayIterator.hasNext()) {
+ JSONObject jsonObject = (JSONObject) jsonArrayIterator.next();
+ String host = (String) jsonObject.remove("ip");
+ mapKnownHosts.put(host, jsonObject);
+ }
+ HostFromPropertiesFileAdapter hfa = new HostFromPropertiesFileAdapter(mapKnownHosts);
+ JSONObject actualMessage = hfa.enrich(new CacheKey("dummy", ip, null));
+ Assert.assertNotNull(actualMessage);
+ Assert.assertEquals(expectedMessage, actualMessage);
+ actualMessage = hfa.enrich(new CacheKey("dummy", ip1, null));
+ JSONObject emptyJson = new JSONObject();
+ Assert.assertEquals(emptyJson, actualMessage);
+ }
+
+
+ @Test
+ public void testInitializeAdapter() throws Exception {
+ Map<String, JSONObject> mapKnownHosts = new HashMap<>();
+ HostFromPropertiesFileAdapter hfa = new HostFromPropertiesFileAdapter(mapKnownHosts);
+ Assert.assertFalse(hfa.initializeAdapter());
+ JSONArray jsonArray = (JSONArray) JSONValue.parse(expectedKnownHostsString);
+ Iterator jsonArrayIterator = jsonArray.iterator();
+ while(jsonArrayIterator.hasNext()) {
+ JSONObject jsonObject = (JSONObject) jsonArrayIterator.next();
+ String host = (String) jsonObject.remove("ip");
+ mapKnownHosts.put(host, jsonObject);
+ }
+ hfa = new HostFromPropertiesFileAdapter(mapKnownHosts);
+ Assert.assertTrue(hfa.initializeAdapter());
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfigTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfigTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfigTest.java
new file mode 100644
index 0000000..2ae2ff0
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/jdbc/MySqlConfigTest.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.jdbc;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class MySqlConfigTest {
+
+ private String sampleURL = "jdbc:mysql://10.22.0.214:3306/GEO?user=root&password=hadoop123";
+ private MySqlConfig conn;
+
+ @Before
+ public void setupJdbc() {
+ conn = new MySqlConfig();
+ conn.setHost("10.22.0.214");
+ conn.setPort(3306);
+ conn.setTable("GEO");
+ conn.setUsername("root");
+ conn.setPassword("hadoop123");
+ }
+
+ @Test
+ public void testGetJdbcUrl() throws Exception {
+ Assert.assertEquals(sampleURL, conn.getJdbcUrl());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
new file mode 100644
index 0000000..a652e95
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.simplehbase;
+
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.domain.SensorEnrichmentConfig;
+import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.hbase.converters.enrichment.EnrichmentKey;
+import org.apache.metron.hbase.converters.enrichment.EnrichmentValue;
+import org.apache.metron.hbase.lookup.EnrichmentLookup;
+import org.apache.metron.hbase.converters.enrichment.EnrichmentHelper;
+import org.apache.metron.integration.util.mock.MockHTable;
+import org.apache.metron.reference.lookup.LookupKV;
+import org.apache.metron.reference.lookup.accesstracker.BloomAccessTracker;
+import org.apache.metron.reference.lookup.accesstracker.PersistentAccessTracker;
+import org.apache.metron.utils.JSONUtils;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+public class SimpleHBaseAdapterTest {
+
+ private String cf = "cf";
+ private String atTableName = "tracker";
+ private final String hbaseTableName = "enrichments";
+ private EnrichmentLookup lookup;
+ private static final String PLAYFUL_CLASSIFICATION_TYPE = "playful_classification";
+ private static final Map<String, String> PLAYFUL_ENRICHMENT = new HashMap<String, String>() {{
+ put("orientation", "north");
+ }};
+
+ /**
+ * {
+ * "10.0.2.3.orientation":"north"
+ * }
+ */
+ @Multiline
+ private String expectedMessageString;
+
+ /**
+ * {
+ * "index": "bro",
+ * "batchSize": 5,
+ * "enrichmentFieldMap": {
+ * "geo": ["ip_dst_addr", "ip_src_addr"],
+ * "host": ["host"]
+ * },
+ * "fieldToEnrichmentTypeMap": {
+ * "ip_dst_addr" : [ "10.0.2.3" ],
+ * "ip_src_addr" : [ "10.3.30.120" ]
+ * }
+ * }
+ */
+ @Multiline
+ private String sourceConfigStr;
+
+ private JSONObject expectedMessage;
+
+ @Before
+ public void setup() throws Exception {
+ final MockHTable trackerTable = (MockHTable) MockHTable.Provider.addToCache(atTableName, cf);
+ final MockHTable hbaseTable = (MockHTable) MockHTable.Provider.addToCache(hbaseTableName, cf);
+ EnrichmentHelper.INSTANCE.load(hbaseTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {{
+ add(new LookupKV<>(new EnrichmentKey("10.0.2.3", "10.0.2.3")
+ , new EnrichmentValue(PLAYFUL_ENRICHMENT)
+ )
+ );
+ }});
+ BloomAccessTracker bat = new BloomAccessTracker(hbaseTableName, 100, 0.03);
+ PersistentAccessTracker pat = new PersistentAccessTracker(hbaseTableName, "0", trackerTable, cf, bat, 0L);
+ lookup = new EnrichmentLookup(hbaseTable, cf, pat);
+ JSONParser jsonParser = new JSONParser();
+ expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString);
+ }
+
+ @Test
+ public void testEnrich() throws Exception {
+ SimpleHBaseAdapter sha = new SimpleHBaseAdapter();
+ sha.lookup = lookup;
+ SensorEnrichmentConfig broSc = JSONUtils.INSTANCE.load(sourceConfigStr, SensorEnrichmentConfig.class);
+ JSONObject actualMessage = sha.enrich(new CacheKey("test", "test", broSc));
+ Assert.assertEquals(actualMessage, new JSONObject());
+ actualMessage = sha.enrich(new CacheKey("ip_dst_addr", "10.0.2.3", broSc));
+ Assert.assertNotNull(actualMessage);
+ Assert.assertEquals(expectedMessage, actualMessage);
+ }
+
+ @Test(expected = Exception.class)
+ public void testInitializeAdapter() {
+ SimpleHBaseConfig config = new SimpleHBaseConfig();
+ SimpleHBaseAdapter sha = new SimpleHBaseAdapter(config);
+ sha.initializeAdapter();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfigTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfigTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfigTest.java
new file mode 100644
index 0000000..832a939
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseConfigTest.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.simplehbase;
+
+import org.apache.metron.hbase.HTableProvider;
+import org.apache.metron.hbase.TableProvider;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SimpleHBaseConfigTest {
+
+
+ private String cf ="cf";
+ private String table = "threatintel";
+ private TableProvider provider;
+
+ @Test
+ public void test(){
+ SimpleHBaseConfig shc = new SimpleHBaseConfig();
+ shc.withHBaseCF(cf);
+ shc.withHBaseTable(table);
+ provider = new HTableProvider();
+ Assert.assertEquals(cf, shc.getHBaseCF());
+ Assert.assertEquals(table, shc.getHBaseTable());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
new file mode 100644
index 0000000..49fe791
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.threatintel;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.metron.domain.SensorEnrichmentConfig;
+import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.hbase.converters.enrichment.EnrichmentKey;
+import org.apache.metron.hbase.converters.enrichment.EnrichmentValue;
+import org.apache.metron.hbase.lookup.EnrichmentLookup;
+import org.apache.metron.hbase.converters.enrichment.EnrichmentHelper;
+import org.apache.metron.integration.util.mock.MockHTable;
+import org.apache.metron.reference.lookup.LookupKV;
+import org.apache.metron.reference.lookup.accesstracker.BloomAccessTracker;
+import org.apache.metron.reference.lookup.accesstracker.PersistentAccessTracker;
+import org.apache.metron.utils.JSONUtils;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+
+public class ThreatIntelAdapterTest {
+
+ public static class ExceptionProvider implements TableProvider {
+
+ public ExceptionProvider() {};
+
+ @Override
+ public HTableInterface getTable(Configuration config, String tableName) throws IOException {
+ throw new IOException();
+ }
+ }
+
+ private String cf = "cf";
+ private String atTableName = "tracker";
+ private static final String MALICIOUS_IP_TYPE = "malicious_ip";
+ private final String threatIntelTableName = "threat_intel";
+ private EnrichmentLookup lookup;
+
+ /**
+ * {
+ * "10.0.2.3":"alert"
+ * }
+ */
+ @Multiline
+ private String expectedMessageString;
+
+ /**
+ * {
+ * "index": "bro",
+ * "batchSize": 5,
+ * "enrichmentFieldMap": {
+ * "geo": ["ip_dst_addr", "ip_src_addr"],
+ * "host": ["host"]
+ * },
+ * "threatIntelFieldMap": {
+ * "hbaseThreatIntel": ["ip_dst_addr", "ip_src_addr"]
+ * },
+ * "fieldToThreatIntelTypeMap": {
+ * "ip_dst_addr" : [ "10.0.2.3" ],
+ * "ip_src_addr" : [ "malicious_ip" ]
+ * }
+ * }
+ */
+ @Multiline
+ private static String sourceConfigStr;
+
+ private JSONObject expectedMessage;
+
+ @Before
+ public void setup() throws Exception {
+
+ final MockHTable trackerTable = (MockHTable) MockHTable.Provider.addToCache(atTableName, cf);
+ final MockHTable threatIntelTable = (MockHTable) MockHTable.Provider.addToCache(threatIntelTableName, cf);
+ EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {{
+ add(new LookupKV<>(new EnrichmentKey("10.0.2.3", "10.0.2.3"), new EnrichmentValue(new HashMap<String, String>())));
+ }});
+
+ BloomAccessTracker bat = new BloomAccessTracker(threatIntelTableName, 100, 0.03);
+ PersistentAccessTracker pat = new PersistentAccessTracker(threatIntelTableName, "0", trackerTable, cf, bat, 0L);
+ lookup = new EnrichmentLookup(threatIntelTable, cf, pat);
+ JSONParser jsonParser = new JSONParser();
+ expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString);
+ }
+
+
+ @Test
+ public void testEnrich() throws Exception {
+ ThreatIntelAdapter tia = new ThreatIntelAdapter();
+ tia.lookup = lookup;
+ SensorEnrichmentConfig broSc = JSONUtils.INSTANCE.load(sourceConfigStr, SensorEnrichmentConfig.class);
+ JSONObject actualMessage = tia.enrich(new CacheKey("ip_dst_addr", "10.0.2.3", broSc));
+ Assert.assertNotNull(actualMessage);
+ Assert.assertEquals(expectedMessage, actualMessage);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testInitializeAdapter() {
+
+ String cf = "cf";
+ String table = "threatintel";
+ String trackCf = "cf";
+ String trackTable = "Track";
+ double falsePositive = 0.03;
+ int expectedInsertion = 1;
+ long millionseconds = (long) 0.1;
+
+ ThreatIntelConfig config = new ThreatIntelConfig();
+ config.withHBaseCF(cf);
+ config.withHBaseTable(table);
+ config.withExpectedInsertions(expectedInsertion);
+ config.withFalsePositiveRate(falsePositive);
+ config.withMillisecondsBetweenPersists(millionseconds);
+ config.withTrackerHBaseCF(trackCf);
+ config.withTrackerHBaseTable(trackTable);
+ config.withProviderImpl(ExceptionProvider.class.getName());
+
+ ThreatIntelAdapter tia = new ThreatIntelAdapter(config);
+ tia.initializeAdapter();
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfigTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfigTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfigTest.java
new file mode 100644
index 0000000..58fd803
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelConfigTest.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.threatintel;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ThreatIntelConfigTest {
+
+ private String cf = "cf";
+ private String table = "threatintel";
+ private String trackCf = "cf";
+ private String trackTable = "Track";
+ private double falsePositive = 0.03;
+ private int expectedInsertion = 1;
+ private long millionseconds = (long) 0.1;
+
+ @Test
+ public void test() {
+ ThreatIntelConfig tic = new ThreatIntelConfig();
+ tic.withHBaseCF(cf);
+ tic.withHBaseTable(table);
+ tic.withExpectedInsertions(expectedInsertion);
+ tic.withFalsePositiveRate(falsePositive);
+ tic.withMillisecondsBetweenPersists(millionseconds);
+ tic.withTrackerHBaseCF(trackCf);
+ tic.withTrackerHBaseTable(trackTable);
+
+ Assert.assertEquals(cf, tic.getHBaseCF());
+ Assert.assertEquals(table, tic.getHBaseTable());
+ Assert.assertEquals(trackCf, tic.getTrackerHBaseCF());
+ Assert.assertEquals(trackTable, tic.getTrackerHBaseTable());
+ Assert.assertEquals(expectedInsertion, tic.getExpectedInsertions());
+ Assert.assertEquals(millionseconds, tic.getMillisecondsBetweenPersists());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/whois/WhoisHBaseAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/whois/WhoisHBaseAdapterTest.java b/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/whois/WhoisHBaseAdapterTest.java
deleted file mode 100644
index 60dd656..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/test/java/org/apache/metron/enrichment/adapters/whois/WhoisHBaseAdapterTest.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.enrichment.adapters.whois;
-
-import java.net.InetAddress;
-import java.util.Properties;
-
-import org.apache.metron.enrichment.bolt.CacheKey;
-import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-import org.json.simple.JSONObject;
-
-import org.apache.metron.test.AbstractTestContext;
-import org.junit.Assert;
-
- /**
- * <ul>
- * <li>Title: </li>
- * <li>Description: </li>
- * <li>Created: Aug 25, 2014 </li>
- * </ul>
- * @version $Revision: 1.1 $
- */
-public class WhoisHBaseAdapterTest extends AbstractTestContext {
-
- private static WhoisHBaseAdapter whoisHbaseAdapter=null;
- private static boolean connected=false;
- /**
- * Constructs a new <code>WhoisHBaseAdapterTest</code> instance.
- * @param name
- */
-
- public WhoisHBaseAdapterTest(String name) {
- super(name);
- }
-
- /**
-
- * @throws java.lang.Exception
- */
- protected static void setUpBeforeClass() throws Exception {
- }
-
- /**
-
- * @throws java.lang.Exception
- */
- protected static void tearDownAfterClass() throws Exception {
- }
-
- /*
- * (non-Javadoc)
- * @see junit.framework.TestCase#setUp()
- */
-
- protected void setUp() throws Exception {
- super.setUp();
- Properties prop = super.getTestProperties();
- Assert.assertNotNull(prop);
-
- if(skipTests(this.getMode())){
- return;//skip tests
- }
-
- String[] zk = prop.get("kafka.zk.list").toString().split(",");
-
- for(String z : zk)
- {
- InetAddress address = InetAddress.getByName(z);
- boolean reachable = address.isReachable(100);
-
- if(!reachable)
- {
- this.setMode("local");
- break;
- //throw new Exception("Unable to reach zookeeper, skipping WHois adapter test");
- }
-
- System.out.println("kafka.zk.list ="+(String) prop.get("kafka.zk.list"));
- System.out.println("kafka.zk.list ="+(String) prop.get("kafka.zk.port"));
- System.out.println("kafka.zk.list ="+(String) prop.get("bolt.enrichment.cif.tablename"));
-
- }
-
- if(skipTests(this.getMode())){
- System.out.println("Local Mode Skipping tests !! ");
- }else{
- whoisHbaseAdapter=new WhoisHBaseAdapter((String)prop.get("bolt.enrichment.whois.hbase.table.name"),(String)prop.get("kafka.zk.list"),(String)prop.get("kafka.zk.port"));
- connected =whoisHbaseAdapter.initializeAdapter();
- Assert.assertTrue(connected);
- }
-
- }
-
- /*
- * (non-Javadoc)
- * @see junit.framework.TestCase#tearDown()
- */
-
- protected void tearDown() throws Exception {
- super.tearDown();
- }
-
- /**
- * Test method for {@link EnrichmentAdapter#initializeAdapter()}.
- */
- public void testInitializeAdapter() {
- if(skipTests(this.getMode())){
- return;//skip tests
- }else{
- Assert.assertTrue(connected);
- }
- }
-
- /**
- * Test method for {@link org.apache.metron.enrichment.adapters.whois.WhoisHBaseAdapter#enrich(CacheKey)}.
- */
- public void testEnrich() {
- if(skipTests(this.getMode())){
- return;//skip tests
- }else{
- JSONObject json = whoisHbaseAdapter.enrich(new CacheKey("whois", "72.163.4.161", null));
-
- //assert Geo Response is not null
- Assert.assertNotNull(json);
-
- //assert LocId is not null
- Assert.assertNotNull(json.get("cisco.com"));
- }
- }
-
-
- /**
- * Returns the whoisHbaseAdapter.
- * @return the whoisHbaseAdapter.
- */
-
- public static WhoisHBaseAdapter getWhoisHbaseAdapter() {
- return whoisHbaseAdapter;
- }
-
- /**
- * Sets the whoisHbaseAdapter.
- * @param whoisHbaseAdapter the whoisHbaseAdapter.
- */
-
- public static void setWhoisHbaseAdapter(WhoisHBaseAdapter whoisHbaseAdapter) {
-
- WhoisHBaseAdapterTest.whoisHbaseAdapter = whoisHbaseAdapter;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-MessageParsers/src/test/java/org/apache/metron/parsing/parsers/GrokParserTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/test/java/org/apache/metron/parsing/parsers/GrokParserTest.java b/metron-streaming/Metron-MessageParsers/src/test/java/org/apache/metron/parsing/parsers/GrokParserTest.java
new file mode 100644
index 0000000..81bf17b
--- /dev/null
+++ b/metron-streaming/Metron-MessageParsers/src/test/java/org/apache/metron/parsing/parsers/GrokParserTest.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsing.parsers;
+
+import com.google.common.collect.MapDifference;
+import com.google.common.collect.Maps;
+import junit.framework.Assert;
+import org.adrianwalker.multilinestring.Multiline;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.junit.Before;
+import org.junit.Test;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class GrokParserTest {
+
+ public String expectedRaw = "2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle";
+
+ /**
+ * {
+ * "roct":0,
+ * "end_reason":"idle",
+ * "ip_dst_addr":"10.0.2.15",
+ * "iflags":"AS",
+ * "rpkt":0,
+ * "original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle",
+ * "tag":0,
+ * "risn":0,
+ * "ip_dst_port":39468,
+ * "ruflags":0,
+ * "app":0,
+ * "protocol":6
+ * ,"isn":"22efa001",
+ * "uflags":0,"duration":"0.000",
+ * "oct":44,
+ * "ip_src_port":80,
+ * "end_time":"2016-01-28 15:29:48.512",
+ * "riflags":0,"start_time":"2016-01-28 15:29:48.512",
+ * "rtt":"0.000",
+ * "rtag":0,
+ * "pkt":1,
+ * "ip_src_addr":"216.21.170.221"
+ * }
+ */
+ @Multiline
+ private String expectedParsedString;
+
+ private JSONObject expectedParsed;
+
+ @Before
+ public void parseJSON() throws ParseException {
+ JSONParser jsonParser = new JSONParser();
+ expectedParsed = (JSONObject) jsonParser.parse(expectedParsedString);
+ }
+
+ @Test
+ public void test() throws IOException, ParseException {
+ String metronHdfsHome = "../Metron-MessageParsers/src/main/resources";
+ String grokHdfsPath = "/patterns/yaf";
+ String patternLabel = "YAF_DELIMITED";
+ GrokParser grokParser = new GrokParser(grokHdfsPath, patternLabel);
+ grokParser.withMetronHDFSHome(metronHdfsHome);
+ grokParser.init();
+ byte[] rawMessage = expectedRaw.getBytes();
+ List<JSONObject> parsedList = grokParser.parse(rawMessage);
+ Assert.assertEquals(1, parsedList.size());
+ compare(expectedParsed, parsedList.get(0));
+ }
+
+ public boolean compare(JSONObject expected, JSONObject actual) {
+ MapDifference mapDifferences = Maps.difference(expected, actual);
+ if (mapDifferences.entriesOnlyOnLeft().size() > 0) Assert.fail("Expected JSON has extra parameters: " + mapDifferences.entriesOnlyOnLeft());
+ if (mapDifferences.entriesOnlyOnRight().size() > 0) Assert.fail("Actual JSON has extra parameters: " + mapDifferences.entriesOnlyOnRight());
+ Map actualDifferences = new HashMap();
+ if (mapDifferences.entriesDiffering().size() > 0) {
+ Map differences = Collections.unmodifiableMap(mapDifferences.entriesDiffering());
+ for (Object key : differences.keySet()) {
+ Object expectedValueObject = expected.get(key);
+ Object actualValueObject = actual.get(key);
+ if (expectedValueObject instanceof Long || expectedValueObject instanceof Integer) {
+ Long expectedValue = Long.parseLong(expectedValueObject.toString());
+ Long actualValue = Long.parseLong(actualValueObject.toString());
+ if (!expectedValue.equals(actualValue)) {
+ actualDifferences.put(key, differences.get(key));
+ }
+ } else {
+ actualDifferences.put(key, differences.get(key));
+ }
+ }
+ }
+ if (actualDifferences.size() > 0) Assert.fail("Expected and Actual JSON values don't match: " + actualDifferences);
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-Testing/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/pom.xml b/metron-streaming/Metron-Testing/pom.xml
index a842322..d06953d 100644
--- a/metron-streaming/Metron-Testing/pom.xml
+++ b/metron-streaming/Metron-Testing/pom.xml
@@ -106,6 +106,11 @@
<artifactId>Metron-MessageParsers</artifactId>
<version>0.1BETA</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>Metron-TestingUtilities</artifactId>
+ <version>0.1BETA</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
index fa140ed..2e4e0cc 100644
--- a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
+++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
@@ -19,18 +19,16 @@ package org.apache.metron.integration;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.*;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.metron.Constants;
import org.apache.metron.domain.Configurations;
-import org.apache.metron.domain.SensorEnrichmentConfig;
import org.apache.metron.hbase.TableProvider;
import org.apache.metron.hbase.converters.enrichment.EnrichmentKey;
import org.apache.metron.hbase.converters.enrichment.EnrichmentValue;
-import org.apache.metron.integration.util.EnrichmentHelper;
+import org.apache.metron.hbase.converters.enrichment.EnrichmentHelper;
import org.apache.metron.integration.util.TestUtils;
import org.apache.metron.integration.util.UnitTestHelper;
import org.apache.metron.integration.util.integration.ComponentRunner;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9ca109b1/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/EnrichmentHelper.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/EnrichmentHelper.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/EnrichmentHelper.java
deleted file mode 100644
index e06a2a5..0000000
--- a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/EnrichmentHelper.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.util;
-
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.metron.hbase.converters.enrichment.EnrichmentConverter;
-import org.apache.metron.hbase.converters.enrichment.EnrichmentKey;
-import org.apache.metron.hbase.converters.enrichment.EnrichmentValue;
-import org.apache.metron.reference.lookup.LookupKV;
-
-import java.io.IOException;
-
-public enum EnrichmentHelper {
- INSTANCE;
- EnrichmentConverter converter = new EnrichmentConverter();
-
- public void load(HTableInterface table, String cf, Iterable<LookupKV<EnrichmentKey, EnrichmentValue>> results) throws IOException {
- for(LookupKV<EnrichmentKey, EnrichmentValue> result : results) {
- Put put = converter.toPut(cf, result.getKey(), result.getValue());
- table.put(put);
- }
- }
-}