You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by si...@apache.org on 2016/01/14 18:04:03 UTC
[68/85] incubator-metron git commit: Move all com/apache folders to
org/apache
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-DataServices/src/main/java/org/apache/metron/services/alerts/Solr_KafkaAlertsService.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/org/apache/metron/services/alerts/Solr_KafkaAlertsService.java b/metron-streaming/Metron-DataServices/src/main/java/org/apache/metron/services/alerts/Solr_KafkaAlertsService.java
new file mode 100644
index 0000000..e6f048f
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/java/org/apache/metron/services/alerts/Solr_KafkaAlertsService.java
@@ -0,0 +1,89 @@
+package com.apache.metron.services.alerts;
+
+import javax.inject.Singleton;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.apache.metron.dataservices.common.MetronService;
+
+@Singleton
+public class Solr_KafkaAlertsService implements MetronService {
+
+ private static final Logger logger = LoggerFactory.getLogger( Solr_KafkaAlertsService.class );
+
+ @Override
+ public String identify() {
+ // TODO Auto-generated method stub
+ return "Elastic Search to Solr Alerts Service";
+ }
+
+ @Override
+ public boolean init(String topicname) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean login() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean registerRulesFromFile() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean registerRules() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public String viewRules() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public boolean editRules() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean deleteRules() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean registerForAlertsTopic(String topicname) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public String receiveAlertAll() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+
+ @Override
+ public boolean disconnectFromAlertsTopic(String topicname) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public String receiveAlertLast() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-DataServices/src/test/java/com/apache/metron/pcapservice/CellTimestampComparatorTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/test/java/com/apache/metron/pcapservice/CellTimestampComparatorTest.java b/metron-streaming/Metron-DataServices/src/test/java/com/apache/metron/pcapservice/CellTimestampComparatorTest.java
deleted file mode 100644
index db38f7c..0000000
--- a/metron-streaming/Metron-DataServices/src/test/java/com/apache/metron/pcapservice/CellTimestampComparatorTest.java
+++ /dev/null
@@ -1,92 +0,0 @@
-package com.apache.metron.pcapservice;
-
-import junit.framework.Assert;
-
-import org.apache.hadoop.hbase.Cell;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import com.apache.metron.pcapservice.CellTimestampComparator;
-
-/**
- * The Class CellTimestampComparatorTest.
- */
-public class CellTimestampComparatorTest {
-
- /**
- * Sets the up.
- *
- * @throws Exception
- * the exception
- */
- @Before
- public void setUp() throws Exception {
- }
-
- /**
- * Tear down.
- *
- * @throws Exception
- * the exception
- */
- @After
- public void tearDown() throws Exception {
- }
-
- /**
- * Test_less.
- */
- @Test
- public void test_less() {
- // mocking
- Cell cell1 = Mockito.mock(Cell.class);
- Mockito.when(cell1.getTimestamp()).thenReturn(13945345808L);
- Cell cell2 = Mockito.mock(Cell.class);
- Mockito.when(cell2.getTimestamp()).thenReturn(13845345808L);
-
- CellTimestampComparator comparator = new CellTimestampComparator();
-
- // actual call and verify
- Assert.assertTrue(comparator.compare(cell2, cell1) == -1);
-
- }
-
- /**
- * Test_greater.
- */
- @Test
- public void test_greater() {
- // mocking
- Cell cell1 = Mockito.mock(Cell.class);
- Mockito.when(cell1.getTimestamp()).thenReturn(13745345808L);
- Cell cell2 = Mockito.mock(Cell.class);
- Mockito.when(cell2.getTimestamp()).thenReturn(13945345808L);
-
- CellTimestampComparator comparator = new CellTimestampComparator();
-
- // actual call and verify
- Assert.assertTrue(comparator.compare(cell2, cell1) == 1);
-
- }
-
- /**
- * Test_equal.
- */
- @Test
- public void test_equal() {
- // mocking
- Cell cell1 = Mockito.mock(Cell.class);
- Mockito.when(cell1.getTimestamp()).thenReturn(13945345808L);
- Cell cell2 = Mockito.mock(Cell.class);
- Mockito.when(cell2.getTimestamp()).thenReturn(13945345808L);
-
- CellTimestampComparator comparator = new CellTimestampComparator();
-
- // actual call and verify
- Assert.assertTrue(comparator.compare(cell2, cell1) == 0);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-DataServices/src/test/java/org/apache/metron/pcapservice/CellTimestampComparatorTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/test/java/org/apache/metron/pcapservice/CellTimestampComparatorTest.java b/metron-streaming/Metron-DataServices/src/test/java/org/apache/metron/pcapservice/CellTimestampComparatorTest.java
new file mode 100644
index 0000000..db38f7c
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/test/java/org/apache/metron/pcapservice/CellTimestampComparatorTest.java
@@ -0,0 +1,92 @@
+package com.apache.metron.pcapservice;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.hbase.Cell;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.apache.metron.pcapservice.CellTimestampComparator;
+
+/**
+ * The Class CellTimestampComparatorTest.
+ */
+public class CellTimestampComparatorTest {
+
+ /**
+ * Sets the up.
+ *
+ * @throws Exception
+ * the exception
+ */
+ @Before
+ public void setUp() throws Exception {
+ }
+
+ /**
+ * Tear down.
+ *
+ * @throws Exception
+ * the exception
+ */
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ /**
+ * Test_less.
+ */
+ @Test
+ public void test_less() {
+ // mocking
+ Cell cell1 = Mockito.mock(Cell.class);
+ Mockito.when(cell1.getTimestamp()).thenReturn(13945345808L);
+ Cell cell2 = Mockito.mock(Cell.class);
+ Mockito.when(cell2.getTimestamp()).thenReturn(13845345808L);
+
+ CellTimestampComparator comparator = new CellTimestampComparator();
+
+ // actual call and verify
+ Assert.assertTrue(comparator.compare(cell2, cell1) == -1);
+
+ }
+
+ /**
+ * Test_greater.
+ */
+ @Test
+ public void test_greater() {
+ // mocking
+ Cell cell1 = Mockito.mock(Cell.class);
+ Mockito.when(cell1.getTimestamp()).thenReturn(13745345808L);
+ Cell cell2 = Mockito.mock(Cell.class);
+ Mockito.when(cell2.getTimestamp()).thenReturn(13945345808L);
+
+ CellTimestampComparator comparator = new CellTimestampComparator();
+
+ // actual call and verify
+ Assert.assertTrue(comparator.compare(cell2, cell1) == 1);
+
+ }
+
+ /**
+ * Test_equal.
+ */
+ @Test
+ public void test_equal() {
+ // mocking
+ Cell cell1 = Mockito.mock(Cell.class);
+ Mockito.when(cell1.getTimestamp()).thenReturn(13945345808L);
+ Cell cell2 = Mockito.mock(Cell.class);
+ Mockito.when(cell2.getTimestamp()).thenReturn(13945345808L);
+
+ CellTimestampComparator comparator = new CellTimestampComparator();
+
+ // actual call and verify
+ Assert.assertTrue(comparator.compare(cell2, cell1) == 0);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java
deleted file mode 100644
index d6034f0..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.enrichment.adapters.cif;
-
-import java.io.Serializable;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-
-public abstract class AbstractCIFAdapter implements EnrichmentAdapter,Serializable{
-
- /**
- *
- */
- private static final long serialVersionUID = -5040559164824221816L;
- protected static final Logger LOG = LoggerFactory
- .getLogger(AbstractCIFAdapter.class);
-
- abstract public boolean initializeAdapter();
- abstract public String enrichByIP(String metadata);
- abstract public String enrichByDomain(String metadata);
- abstract public String enrichByEmail(String metadata);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java
deleted file mode 100644
index 5c31a3b..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.enrichment.adapters.cif;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.json.simple.JSONObject;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.log4j.Logger;
-
-@SuppressWarnings("unchecked")
-public class CIFHbaseAdapter extends AbstractCIFAdapter {
-
- private static final long serialVersionUID = 1L;
- private String _tableName;
- private HTableInterface table;
- private String _quorum;
- private String _port;
-
- public CIFHbaseAdapter(String quorum, String port, String tableName) {
- _quorum = quorum;
- _port = port;
- _tableName = tableName;
- }
-
- /** The LOGGER. */
- private static final Logger LOGGER = Logger
- .getLogger(CIFHbaseAdapter.class);
-
- public JSONObject enrich(String metadata) {
-
- JSONObject output = new JSONObject();
- LOGGER.debug("=======Looking Up For:" + metadata);
- output.putAll(getCIFObject(metadata));
-
- return output;
- }
-
- @SuppressWarnings({ "rawtypes", "deprecation" })
- protected Map getCIFObject(String key) {
-
- LOGGER.debug("=======Pinging HBase For:" + key);
-
- Get get = new Get(key.getBytes());
- Result rs;
- Map output = new HashMap();
-
- try {
- rs = table.get(get);
-
- for (KeyValue kv : rs.raw())
- output.put(new String(kv.getQualifier()), "Y");
-
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- return output;
- }
-
- @Override
- public boolean initializeAdapter() {
-
- // Initialize HBase Table
- Configuration conf = null;
- conf = HBaseConfiguration.create();
- conf.set("hbase.zookeeper.quorum", _quorum);
- conf.set("hbase.zookeeper.property.clientPort", _port);
-
- try {
- LOGGER.debug("=======Connecting to HBASE===========");
- LOGGER.debug("=======ZOOKEEPER = "
- + conf.get("hbase.zookeeper.quorum"));
- HConnection connection = HConnectionManager.createConnection(conf);
- table = connection.getTable(_tableName);
- return true;
- } catch (IOException e) {
- // TODO Auto-generated catch block
- LOGGER.debug("=======Unable to Connect to HBASE===========");
- e.printStackTrace();
- }
-
- return false;
- }
-
- @Override
- public String enrichByIP(String metadata) {
- return null;
- }
-
- @Override
- public String enrichByDomain(String metadata) {
- return null;
- }
-
- @Override
- public String enrichByEmail(String metadata) {
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/geo/AbstractGeoAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/geo/AbstractGeoAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/geo/AbstractGeoAdapter.java
deleted file mode 100644
index 717bebb..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/geo/AbstractGeoAdapter.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.enrichment.adapters.geo;
-
-import java.io.Serializable;
-import java.net.InetAddress;
-
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.apache.metron.enrichment.common.GenericEnrichmentBolt;
-import com.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-
-@SuppressWarnings("serial")
-public abstract class AbstractGeoAdapter implements EnrichmentAdapter,
- Serializable {
-
- protected static final Logger _LOG = LoggerFactory
- .getLogger(GenericEnrichmentBolt.class);
-
- abstract public JSONObject enrich(String metadata);
-
- abstract public boolean initializeAdapter();
-
- /**
- * Check if we can reach the IP where geo data is storred
- *
- * @param ip
- * - ip of geo database
- * @param timeout
- * - timeout for a connection attempt
- * @return - True if can connect, False if cannot
- * @throws Exception
- */
- public boolean checkIfReachable(String ip, int timeout) throws Exception {
- boolean reachable = InetAddress.getByName(ip).isReachable(timeout);
-
- if (!reachable)
- return false;
-
- return true;
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/geo/GeoMysqlAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/geo/GeoMysqlAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/geo/GeoMysqlAdapter.java
deleted file mode 100644
index da9b063..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/geo/GeoMysqlAdapter.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.enrichment.adapters.geo;
-
-import java.net.InetAddress;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.Statement;
-
-import org.apache.commons.validator.routines.InetAddressValidator;
-import org.json.simple.JSONObject;
-
-@SuppressWarnings("serial")
-public class GeoMysqlAdapter extends AbstractGeoAdapter {
-
- private Connection connection = null;
- private Statement statement = null;
- private String _ip;
- private String _username;
- private String _password;
- private String _tablename;
- private InetAddressValidator ipvalidator = new InetAddressValidator();
-
- public GeoMysqlAdapter(String ip, int port, String username,
- String password, String tablename) {
- try {
- _ip = InetAddress.getByName(ip).getHostAddress();
-
- boolean reachable = checkIfReachable(ip, 500);
-
- if (!reachable)
- throw new Exception("Unable to reach IP " + _ip
- + " with username " + _username + " and password "
- + _password + " accessing table name " + _tablename);
-
- } catch (Exception e) {
- _LOG.error("Environment misconfigured, cannot reach MYSQL server....");
- e.printStackTrace();
- }
-
- _username = username;
- _password = password;
- _tablename = tablename;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public JSONObject enrich(String metadata) {
-
- ResultSet resultSet = null;
-
- try {
-
- _LOG.trace("[Metron] Received metadata: " + metadata);
-
- InetAddress addr = InetAddress.getByName(metadata);
-
- if (addr.isAnyLocalAddress() || addr.isLoopbackAddress()
- || addr.isSiteLocalAddress() || addr.isMulticastAddress()
- || !ipvalidator.isValidInet4Address(metadata)) {
- _LOG.trace("[Metron] Not a remote IP: " + metadata);
- _LOG.trace("[Metron] Returning enrichment: " + "{}");
-
- return new JSONObject();
- }
-
- _LOG.trace("[Metron] Is a valid remote IP: " + metadata);
-
- statement = connection.createStatement(
- ResultSet.TYPE_SCROLL_INSENSITIVE,
- ResultSet.CONCUR_READ_ONLY);
- String locid_query = "select IPTOLOCID(\"" + metadata
- + "\") as ANS";
- resultSet = statement.executeQuery(locid_query);
-
- if (resultSet == null)
- throw new Exception("Invalid result set for metadata: "
- + metadata + ". Query run was: " + locid_query);
-
- resultSet.last();
- int size = resultSet.getRow();
-
- if (size == 0)
- throw new Exception("No result returned for: " + metadata
- + ". Query run was: " + locid_query);
-
- resultSet.beforeFirst();
- resultSet.next();
-
- String locid = null;
- locid = resultSet.getString("ANS");
-
- if (locid == null)
- throw new Exception("Invalid location id for: " + metadata
- + ". Query run was: " + locid_query);
-
- String geo_query = "select * from location where locID = " + locid
- + ";";
- resultSet = statement.executeQuery(geo_query);
-
- if (resultSet == null)
- throw new Exception(
- "Invalid result set for metadata and locid: "
- + metadata + ", " + locid + ". Query run was: "
- + geo_query);
-
- resultSet.last();
- size = resultSet.getRow();
-
- if (size == 0)
- throw new Exception(
- "No result id returned for metadata and locid: "
- + metadata + ", " + locid + ". Query run was: "
- + geo_query);
-
- resultSet.beforeFirst();
- resultSet.next();
-
- JSONObject jo = new JSONObject();
- jo.put("locID", resultSet.getString("locID"));
- jo.put("country", resultSet.getString("country"));
- jo.put("city", resultSet.getString("city"));
- jo.put("postalCode", resultSet.getString("postalCode"));
- jo.put("latitude", resultSet.getString("latitude"));
- jo.put("longitude", resultSet.getString("longitude"));
- jo.put("dmaCode", resultSet.getString("dmaCode"));
- jo.put("locID", resultSet.getString("locID"));
-
- jo.put("location_point", jo.get("longitude") + "," + jo.get("latitude"));
-
- _LOG.debug("Returning enrichment: " + jo);
-
- return jo;
-
- } catch (Exception e) {
- e.printStackTrace();
- _LOG.error("Enrichment failure: " + e);
- return new JSONObject();
- }
- }
-
- @Override
- public boolean initializeAdapter() {
-
- _LOG.info("[Metron] Initializing MysqlAdapter....");
-
- try {
-
- Class.forName("com.mysql.jdbc.Driver");
- connection = DriverManager.getConnection("jdbc:mysql://" + _ip
- + "/" + _tablename + "?user=" + _username + "&password="
- + _password);
-
- connection.setReadOnly(true);
-
- if (!connection.isValid(0))
- throw new Exception("Invalid connection string....");
-
- _LOG.info("[Metron] Set JDBC connection....");
-
- return true;
- } catch (Exception e) {
- e.printStackTrace();
- _LOG.error("[Metron] JDBC connection failed....");
-
- return false;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java
deleted file mode 100644
index e3c878d..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.enrichment.adapters.host;
-
-import java.io.Serializable;
-
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-
-public abstract class AbstractHostAdapter implements EnrichmentAdapter,Serializable{
-
- /**
- * Adapter to attach reputation information to the telemetry message
- */
- private static final long serialVersionUID = 8280523289446309728L;
- protected static final Logger LOG = LoggerFactory
- .getLogger(AbstractHostAdapter.class);
-
- abstract public boolean initializeAdapter();
- abstract public JSONObject enrich(String metadata);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java
deleted file mode 100644
index e5efdce..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.enrichment.adapters.host;
-
-import java.util.Map;
-
-import org.json.simple.JSONObject;
-
-@SuppressWarnings("serial")
-public class HostFromPropertiesFileAdapter extends AbstractHostAdapter {
-
- Map<String, JSONObject> _known_hosts;
-
- public HostFromPropertiesFileAdapter(Map<String, JSONObject> known_hosts)
- {
- _known_hosts = known_hosts;
- }
-
- @Override
- public boolean initializeAdapter()
- {
-
- if(_known_hosts.size() > 0)
- return true;
- else
- return false;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public JSONObject enrich(String metadata) {
-
-
- if(!_known_hosts.containsKey(metadata))
- return new JSONObject();
-
- JSONObject enrichment = new JSONObject();
- enrichment.put("known_info", (JSONObject) _known_hosts.get(metadata));
- return enrichment;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/threat/AbstractThreatAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/threat/AbstractThreatAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/threat/AbstractThreatAdapter.java
deleted file mode 100644
index b12cbb7..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/threat/AbstractThreatAdapter.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.enrichment.adapters.threat;
-
-import java.io.Serializable;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-
-public abstract class AbstractThreatAdapter implements EnrichmentAdapter,Serializable{
-
-
- private static final long serialVersionUID = 1524030932856141771L;
- protected static final Logger LOG = LoggerFactory
- .getLogger(AbstractThreatAdapter.class);
-
- abstract public boolean initializeAdapter();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/threat/ThreatHbaseAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/threat/ThreatHbaseAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/threat/ThreatHbaseAdapter.java
deleted file mode 100644
index c08b87b..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/threat/ThreatHbaseAdapter.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.enrichment.adapters.threat;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-import org.json.simple.parser.ParseException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.log4j.Logger;
-
-@SuppressWarnings("unchecked")
-public class ThreatHbaseAdapter extends AbstractThreatAdapter {
-
- private static final long serialVersionUID = 1L;
- private String _tableName;
- private HTableInterface table;
- private String _quorum;
- private String _port;
-
- public ThreatHbaseAdapter(String quorum, String port, String tableName) {
- _quorum = quorum;
- _port = port;
- _tableName = tableName;
- }
-
- /** The LOGGER. */
- private static final Logger LOGGER = Logger
- .getLogger(ThreatHbaseAdapter.class);
-
- public JSONObject enrich(String metadata) {
-
- JSONObject output = new JSONObject();
- LOGGER.debug("=======Looking Up For:" + metadata);
- output.putAll(getThreatObject(metadata));
-
- return output;
- }
-
- @SuppressWarnings({ "rawtypes", "deprecation" })
- protected Map getThreatObject(String key) {
-
- LOGGER.debug("=======Pinging HBase For:" + key);
-
- Get get = new Get(Bytes.toBytes(key));
- Result rs;
- Map output = new HashMap();
-
- try {
- rs = table.get(get);
-
- if (!rs.isEmpty()) {
- byte[] source_family = Bytes.toBytes("source");
- JSONParser parser = new JSONParser();
-
- Map<byte[], byte[]> sourceFamilyMap = rs.getFamilyMap(source_family);
-
- for (Map.Entry<byte[], byte[]> entry : sourceFamilyMap.entrySet()) {
- String k = Bytes.toString(entry.getKey());
- LOGGER.debug("=======Found intel from source: " + k);
- output.put(k,parser.parse(Bytes.toString(entry.getValue())));
- }
- }
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } catch (ParseException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- return output;
- }
-
- @Override
- public boolean initializeAdapter() {
-
- // Initialize HBase Table
- Configuration conf = null;
- conf = HBaseConfiguration.create();
- conf.set("hbase.zookeeper.quorum", _quorum);
- conf.set("hbase.zookeeper.property.clientPort", _port);
-
- try {
- LOGGER.debug("=======Connecting to HBASE===========");
- LOGGER.debug("=======ZOOKEEPER = "
- + conf.get("hbase.zookeeper.quorum"));
- HConnection connection = HConnectionManager.createConnection(conf);
- table = connection.getTable(_tableName);
- return true;
- } catch (IOException e) {
- // TODO Auto-generated catch block
- LOGGER.debug("=======Unable to Connect to HBASE===========");
- e.printStackTrace();
- }
-
- return false;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/whois/AbstractWhoisAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/whois/AbstractWhoisAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/whois/AbstractWhoisAdapter.java
deleted file mode 100644
index 89b2e55..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/whois/AbstractWhoisAdapter.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.enrichment.adapters.whois;
-
-import java.io.Serializable;
-
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-
-public abstract class AbstractWhoisAdapter implements EnrichmentAdapter,Serializable{
-
- /**
- *
- */
- private static final long serialVersionUID = 8280523289446309728L;
- protected static final Logger LOG = LoggerFactory
- .getLogger(AbstractWhoisAdapter.class);
-
- abstract public boolean initializeAdapter();
- abstract public JSONObject enrich(String metadata);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/whois/WhoisHBaseAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/whois/WhoisHBaseAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/whois/WhoisHBaseAdapter.java
deleted file mode 100644
index 88f2810..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/adapters/whois/WhoisHBaseAdapter.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.enrichment.adapters.whois;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.json.simple.JSONObject;
-
-import com.google.common.base.Joiner;
-import com.apache.metron.tldextractor.BasicTldExtractor;
-
-public class WhoisHBaseAdapter extends AbstractWhoisAdapter {
-
- /**
- *
- */
- private static final long serialVersionUID = 3371873619030870389L;
- private HTableInterface table;
- private String _table_name;
- private String _quorum;
- private String _port;
- private BasicTldExtractor tldex = new BasicTldExtractor();
-
- public WhoisHBaseAdapter(String table_name, String quorum, String port) {
- _table_name = table_name;
- _quorum = quorum;
- _port = port;
- }
-
- public boolean initializeAdapter() {
- Configuration conf = null;
- conf = HBaseConfiguration.create();
- conf.set("hbase.zookeeper.quorum", _quorum);
- conf.set("hbase.zookeeper.property.clientPort", _port);
- conf.set("zookeeper.session.timeout", "20");
- conf.set("hbase.rpc.timeout", "20");
- conf.set("zookeeper.recovery.retry", "1");
- conf.set("zookeeper.recovery.retry.intervalmill", "1");
-
- try {
-
- LOG.trace("[Metron] Connecting to HBase");
- LOG.trace("[Metron] ZOOKEEPER = "
- + conf.get("hbase.zookeeper.quorum"));
-
- LOG.trace("[Metron] CONNECTING TO HBASE WITH: " + conf);
-
- HConnection connection = HConnectionManager.createConnection(conf);
-
- LOG.trace("[Metron] CONNECTED TO HBASE");
-
- table = connection.getTable(_table_name);
-
- LOG.trace("--------CONNECTED TO TABLE: " + table);
-
- JSONObject tester = enrich("cisco.com");
-
- if (tester.keySet().size() == 0)
- throw new IOException(
- "Either HBASE is misconfigured or whois table is missing");
-
- return true;
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- return false;
-
- }
-
- @SuppressWarnings({ "unchecked", "deprecation" })
- public JSONObject enrich(String metadataIn) {
-
- String metadata = tldex.extract2LD(metadataIn);
-
- LOG.trace("[Metron] Pinging HBase For:" + metadata);
-
-
- JSONObject output = new JSONObject();
- JSONObject payload = new JSONObject();
-
- Get get = new Get(metadata.getBytes());
- Result rs;
-
- try {
- rs = table.get(get);
-
- for (KeyValue kv : rs.raw())
- payload.put(metadata, new String(kv.getValue()));
-
- output.put("whois", payload);
-
- } catch (IOException e) {
- payload.put(metadata, "{}");
- output.put("whois", payload);
- e.printStackTrace();
- }
-
- return output;
-
- }
-
-// private String format(String input) {
-// String output = input;
-// String[] tokens = input.split("\\.");
-// if(tokens.length > 2) {
-// output = Joiner.on(".").join(Arrays.copyOfRange(tokens, 1, tokens.length));;
-// }
-// return output;
-// }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/common/AbstractEnrichmentBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/common/AbstractEnrichmentBolt.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/common/AbstractEnrichmentBolt.java
deleted file mode 100644
index b301592..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/common/AbstractEnrichmentBolt.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.apache.metron.enrichment.common;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.base.BaseRichBolt;
-
-import com.codahale.metrics.Counter;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-import com.apache.metron.metrics.MetricReporter;
-
-@SuppressWarnings("rawtypes")
-public abstract class AbstractEnrichmentBolt extends BaseRichBolt {
- /**
- * Abstract enrichment bolt
- */
- private static final long serialVersionUID = -6710596708304282838L;
-
- protected static final Logger LOG = LoggerFactory
- .getLogger(AbstractEnrichmentBolt.class);
-
- protected OutputCollector _collector;
- protected String _OutputFieldName;
-
- protected String _enrichment_tag;
- protected Long _MAX_CACHE_SIZE_OBJECTS_NUM;
- protected Long _MAX_TIME_RETAIN_MINUTES;
-
- // JSON Keys to be enriched
- protected List<String> _jsonKeys;
- protected EnrichmentAdapter _adapter;
- protected MetricReporter _reporter;
-
- protected transient CacheLoader<String, JSONObject> loader;
- protected transient LoadingCache<String, JSONObject> cache;
-
- protected Counter ackCounter, emitCounter, failCounter;
-
- protected void registerCounters() {
-
- String ackString = _adapter.getClass().getSimpleName() + ".ack";
-
- String emitString = _adapter.getClass().getSimpleName() + ".emit";
-
- String failString = _adapter.getClass().getSimpleName() + ".fail";
-
- ackCounter = _reporter.registerCounter(ackString);
- emitCounter = _reporter.registerCounter(emitString);
- failCounter = _reporter.registerCounter(failString);
-
- }
-
- public final void prepare(Map conf, TopologyContext topologyContext,
- OutputCollector collector) {
- _collector = collector;
-
- if (this._OutputFieldName == null)
- throw new IllegalStateException("OutputFieldName must be specified");
- if (this._enrichment_tag == null)
- throw new IllegalStateException("enrichment_tag must be specified");
- if (this._MAX_CACHE_SIZE_OBJECTS_NUM == null)
- throw new IllegalStateException("MAX_CACHE_SIZE_OBJECTS_NUM must be specified");
- if (this._MAX_TIME_RETAIN_MINUTES == null)
- throw new IllegalStateException("MAX_TIME_RETAIN_MINUTES must be specified");
- if (this._adapter == null)
- throw new IllegalStateException("Adapter must be specified");
- if (this._jsonKeys == null)
- throw new IllegalStateException(
- "JSON Keys to be enriched, must be specified");
-
- loader = new CacheLoader<String, JSONObject>() {
- public JSONObject load(String key) throws Exception {
- return _adapter.enrich(key);
- }
- };
-
- cache = CacheBuilder.newBuilder().maximumSize(_MAX_CACHE_SIZE_OBJECTS_NUM)
- .expireAfterWrite(_MAX_TIME_RETAIN_MINUTES, TimeUnit.MINUTES)
- .build(loader);
-
- boolean success = _adapter.initializeAdapter();
-
- if (!success) {
- LOG.error("[Metron] EnrichmentBolt could not initialize adapter");
- throw new IllegalStateException("Could not initialize adapter...");
- }
-
- try {
- doPrepare(conf, topologyContext, collector);
- } catch (IOException e) {
- LOG.error("[Metron] Counld not initialize...");
- e.printStackTrace();
- }
-
- }
-
- abstract void doPrepare(Map conf, TopologyContext topologyContext,
- OutputCollector collector) throws IOException;
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/common/GenericEnrichmentBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/common/GenericEnrichmentBolt.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/common/GenericEnrichmentBolt.java
deleted file mode 100644
index 6e91e89..0000000
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/com/apache/metron/enrichment/common/GenericEnrichmentBolt.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.apache.metron.enrichment.common;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.configuration.Configuration;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Fields;
-import backtype.storm.tuple.Tuple;
-import backtype.storm.tuple.Values;
-
-import com.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-import com.apache.metron.helpers.topology.ErrorGenerator;
-import com.apache.metron.json.serialization.JSONEncoderHelper;
-import com.apache.metron.metrics.MetricReporter;
-
-/**
- * Uses an adapter to enrich telemetry messages with additional metadata
- * entries. For a list of available enrichment adapters see
- * com.apache.metron.enrichment.adapters.
- * <p>
- * At the moment of release the following enrichment adapters are available:
- * <p>
- * <ul>
- *
- * <li>geo = attaches geo coordinates to IPs
- * <li>whois = attaches whois information to domains
- * <li>host = attaches reputation information to known hosts
- * <li>CIF = attaches information from threat intelligence feeds
- * <ul>
- * <p>
- * <p>
- * Enrichments are optional
- **/
-
-@SuppressWarnings({ "rawtypes", "serial" })
-public class GenericEnrichmentBolt extends AbstractEnrichmentBolt {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(GenericEnrichmentBolt.class);
- private JSONObject metricConfiguration;
-
- /**
- * @param adapter
- * Adapter for doing the enrichment
- * @return Instance of this class
- */
-
- public GenericEnrichmentBolt withAdapter(EnrichmentAdapter adapter) {
- _adapter = adapter;
- return this;
- }
-
- /**
- * @param OutputFieldName
- * Fieldname of the output tuple for this bolt
- * @return Instance of this class
- */
-
- public GenericEnrichmentBolt withOutputFieldName(String OutputFieldName) {
- _OutputFieldName = OutputFieldName;
- return this;
- }
-
- /**
- * @param EnrichmentTag
- * Defines what tag the enrichment will be tagged with in the
- * telemetry message
- * @return Instance of this class
- */
-
- public GenericEnrichmentBolt withEnrichmentTag(String EnrichmentTag) {
- _enrichment_tag = EnrichmentTag;
- return this;
- }
-
- /**
- * @param MAX_CACHE_SIZE_OBJECTS_NUM
- * Maximum size of cache before flushing
- * @return Instance of this class
- */
-
- public GenericEnrichmentBolt withMaxCacheSize(long MAX_CACHE_SIZE_OBJECTS_NUM) {
- _MAX_CACHE_SIZE_OBJECTS_NUM = MAX_CACHE_SIZE_OBJECTS_NUM;
- return this;
- }
-
- /**
- * @param MAX_TIME_RETAIN_MINUTES
- * Maximum time to retain cached entry before expiring
- * @return Instance of this class
- */
-
- public GenericEnrichmentBolt withMaxTimeRetain(long MAX_TIME_RETAIN_MINUTES) {
- _MAX_TIME_RETAIN_MINUTES = MAX_TIME_RETAIN_MINUTES;
- return this;
- }
-
- /**
- * @param jsonKeys
- * Keys in the telemetry message that are to be enriched by this
- * bolt
- * @return Instance of this class
- */
-
- public GenericEnrichmentBolt withKeys(List<String> jsonKeys) {
- _jsonKeys = jsonKeys;
- return this;
- }
-
- /**
- * @param config
- * A class for generating custom metrics into graphite
- * @return Instance of this class
- */
-
- public GenericEnrichmentBolt withMetricConfiguration(Configuration config) {
- this.metricConfiguration = JSONEncoderHelper.getJSON(config
- .subset("com.apache.metron.metrics"));
- return this;
- }
-
- @SuppressWarnings("unchecked")
- public void execute(Tuple tuple) {
-
- LOG.trace("[Metron] Starting enrichment");
-
- JSONObject in_json = null;
- String key = null;
-
- try {
-
- key = tuple.getStringByField("key");
- in_json = (JSONObject) tuple.getValueByField("message");
-
- if (in_json == null || in_json.isEmpty())
- throw new Exception("Could not parse binary stream to JSON");
-
- if(key == null)
- throw new Exception("Key is not valid");
-
- LOG.trace("[Metron] Received tuple: " + in_json);
-
- JSONObject message = (JSONObject) in_json.get("message");
-
- if (message == null || message.isEmpty())
- throw new Exception("Could not extract message from JSON: "
- + in_json);
-
- LOG.trace("[Metron] Extracted message: " + message);
-
- for (String jsonkey : _jsonKeys) {
- LOG.trace("[Metron] Processing:" + jsonkey + " within:"
- + message);
-
- String jsonvalue = (String) message.get(jsonkey);
- LOG.trace("[Metron] Processing: " + jsonkey + " -> "
- + jsonvalue);
-
- if (null == jsonvalue) {
- LOG.trace("[Metron] Key " + jsonkey
- + "not present in message " + message);
- continue;
- }
-
- // If the field is empty, no need to enrich
- if ( jsonvalue.length() == 0) {
- continue;
- }
-
- JSONObject enrichment = cache.getUnchecked(jsonvalue);
- LOG.trace("[Metron] Enriched: " + jsonkey + " -> "
- + enrichment);
-
- if (enrichment == null)
- throw new Exception("[Metron] Could not enrich string: "
- + jsonvalue);
-
- if (!in_json.containsKey("enrichment")) {
- in_json.put("enrichment", new JSONObject());
- LOG.trace("[Metron] Starting a string of enrichments");
- }
-
- JSONObject enr1 = (JSONObject) in_json.get("enrichment");
-
- if (enr1 == null)
- throw new Exception("Internal enrichment is empty");
-
- if (!enr1.containsKey(_enrichment_tag)) {
- enr1.put(_enrichment_tag, new JSONObject());
- LOG.trace("[Metron] Starting a new enrichment");
- }
-
- LOG.trace("[Metron] ENR1 is: " + enr1);
-
- JSONObject enr2 = (JSONObject) enr1.get(_enrichment_tag);
- enr2.put(jsonkey, enrichment);
-
- LOG.trace("[Metron] ENR2 is: " + enr2);
-
- enr1.put(_enrichment_tag, enr2);
- in_json.put("enrichment", enr1);
- }
-
- LOG.debug("[Metron] Generated combined enrichment: " + in_json);
-
- _collector.emit("message", new Values(key, in_json));
- _collector.ack(tuple);
-
- if (_reporter != null) {
- emitCounter.inc();
- ackCounter.inc();
- }
- } catch (Exception e) {
-
- LOG.error("[Metron] Unable to enrich message: " + in_json);
- _collector.fail(tuple);
-
- if (_reporter != null) {
- failCounter.inc();
- }
-
- JSONObject error = ErrorGenerator.generateErrorMessage("Enrichment problem: " + in_json, e);
- _collector.emit("error", new Values(error));
- }
-
-
-
- }
-
- public void declareOutputFields(OutputFieldsDeclarer declearer) {
- declearer.declareStream("message", new Fields("key", "message"));
- declearer.declareStream("error", new Fields("message"));
- }
-
- @Override
- void doPrepare(Map conf, TopologyContext topologyContext,
- OutputCollector collector) throws IOException {
- LOG.info("[Metron] Preparing Enrichment Bolt...");
-
- _collector = collector;
-
- try {
- _reporter = new MetricReporter();
- _reporter.initialize(metricConfiguration,
- GenericEnrichmentBolt.class);
- this.registerCounters();
- } catch (Exception e) {
- LOG.info("[Metron] Unable to initialize metrics reporting");
- }
-
- LOG.info("[Metron] Enrichment bolt initialized...");
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java
new file mode 100644
index 0000000..d6034f0
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/cif/AbstractCIFAdapter.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apache.metron.enrichment.adapters.cif;
+
+import java.io.Serializable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.apache.metron.enrichment.interfaces.EnrichmentAdapter;
+
+public abstract class AbstractCIFAdapter implements EnrichmentAdapter,Serializable{
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = -5040559164824221816L;
+ protected static final Logger LOG = LoggerFactory
+ .getLogger(AbstractCIFAdapter.class);
+
+ abstract public boolean initializeAdapter();
+ abstract public String enrichByIP(String metadata);
+ abstract public String enrichByDomain(String metadata);
+ abstract public String enrichByEmail(String metadata);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java
new file mode 100644
index 0000000..5c31a3b
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/cif/CIFHbaseAdapter.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apache.metron.enrichment.adapters.cif;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.json.simple.JSONObject;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.log4j.Logger;
+
+@SuppressWarnings("unchecked")
+public class CIFHbaseAdapter extends AbstractCIFAdapter {
+
+ private static final long serialVersionUID = 1L;
+ private String _tableName;
+ private HTableInterface table;
+ private String _quorum;
+ private String _port;
+
+ public CIFHbaseAdapter(String quorum, String port, String tableName) {
+ _quorum = quorum;
+ _port = port;
+ _tableName = tableName;
+ }
+
+ /** The LOGGER. */
+ private static final Logger LOGGER = Logger
+ .getLogger(CIFHbaseAdapter.class);
+
+ public JSONObject enrich(String metadata) {
+
+ JSONObject output = new JSONObject();
+ LOGGER.debug("=======Looking Up For:" + metadata);
+ output.putAll(getCIFObject(metadata));
+
+ return output;
+ }
+
+ @SuppressWarnings({ "rawtypes", "deprecation" })
+ protected Map getCIFObject(String key) {
+
+ LOGGER.debug("=======Pinging HBase For:" + key);
+
+ Get get = new Get(key.getBytes());
+ Result rs;
+ Map output = new HashMap();
+
+ try {
+ rs = table.get(get);
+
+ for (KeyValue kv : rs.raw())
+ output.put(new String(kv.getQualifier()), "Y");
+
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ return output;
+ }
+
+ @Override
+ public boolean initializeAdapter() {
+
+ // Initialize HBase Table
+ Configuration conf = null;
+ conf = HBaseConfiguration.create();
+ conf.set("hbase.zookeeper.quorum", _quorum);
+ conf.set("hbase.zookeeper.property.clientPort", _port);
+
+ try {
+ LOGGER.debug("=======Connecting to HBASE===========");
+ LOGGER.debug("=======ZOOKEEPER = "
+ + conf.get("hbase.zookeeper.quorum"));
+ HConnection connection = HConnectionManager.createConnection(conf);
+ table = connection.getTable(_tableName);
+ return true;
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ LOGGER.debug("=======Unable to Connect to HBASE===========");
+ e.printStackTrace();
+ }
+
+ return false;
+ }
+
+ @Override
+ public String enrichByIP(String metadata) {
+ return null;
+ }
+
+ @Override
+ public String enrichByDomain(String metadata) {
+ return null;
+ }
+
+ @Override
+ public String enrichByEmail(String metadata) {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/geo/AbstractGeoAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/geo/AbstractGeoAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/geo/AbstractGeoAdapter.java
new file mode 100644
index 0000000..717bebb
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/geo/AbstractGeoAdapter.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apache.metron.enrichment.adapters.geo;
+
+import java.io.Serializable;
+import java.net.InetAddress;
+
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.apache.metron.enrichment.common.GenericEnrichmentBolt;
+import com.apache.metron.enrichment.interfaces.EnrichmentAdapter;
+
+@SuppressWarnings("serial")
+public abstract class AbstractGeoAdapter implements EnrichmentAdapter,
+ Serializable {
+
+ protected static final Logger _LOG = LoggerFactory
+ .getLogger(GenericEnrichmentBolt.class);
+
+ abstract public JSONObject enrich(String metadata);
+
+ abstract public boolean initializeAdapter();
+
+ /**
+ * Check if we can reach the IP where geo data is storred
+ *
+ * @param ip
+ * - ip of geo database
+ * @param timeout
+ * - timeout for a connection attempt
+ * @return - True if can connect, False if cannot
+ * @throws Exception
+ */
+ public boolean checkIfReachable(String ip, int timeout) throws Exception {
+ boolean reachable = InetAddress.getByName(ip).isReachable(timeout);
+
+ if (!reachable)
+ return false;
+
+ return true;
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoMysqlAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoMysqlAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoMysqlAdapter.java
new file mode 100644
index 0000000..da9b063
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoMysqlAdapter.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apache.metron.enrichment.adapters.geo;
+
+import java.net.InetAddress;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import org.apache.commons.validator.routines.InetAddressValidator;
+import org.json.simple.JSONObject;
+
+@SuppressWarnings("serial")
+public class GeoMysqlAdapter extends AbstractGeoAdapter {
+
+ private Connection connection = null;
+ private Statement statement = null;
+ private String _ip;
+ private String _username;
+ private String _password;
+ private String _tablename;
+ private InetAddressValidator ipvalidator = new InetAddressValidator();
+
+ public GeoMysqlAdapter(String ip, int port, String username,
+ String password, String tablename) {
+ try {
+ _ip = InetAddress.getByName(ip).getHostAddress();
+
+ boolean reachable = checkIfReachable(ip, 500);
+
+ if (!reachable)
+ throw new Exception("Unable to reach IP " + _ip
+ + " with username " + _username + " and password "
+ + _password + " accessing table name " + _tablename);
+
+ } catch (Exception e) {
+ _LOG.error("Environment misconfigured, cannot reach MYSQL server....");
+ e.printStackTrace();
+ }
+
+ _username = username;
+ _password = password;
+ _tablename = tablename;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public JSONObject enrich(String metadata) {
+
+ ResultSet resultSet = null;
+
+ try {
+
+ _LOG.trace("[Metron] Received metadata: " + metadata);
+
+ InetAddress addr = InetAddress.getByName(metadata);
+
+ if (addr.isAnyLocalAddress() || addr.isLoopbackAddress()
+ || addr.isSiteLocalAddress() || addr.isMulticastAddress()
+ || !ipvalidator.isValidInet4Address(metadata)) {
+ _LOG.trace("[Metron] Not a remote IP: " + metadata);
+ _LOG.trace("[Metron] Returning enrichment: " + "{}");
+
+ return new JSONObject();
+ }
+
+ _LOG.trace("[Metron] Is a valid remote IP: " + metadata);
+
+ statement = connection.createStatement(
+ ResultSet.TYPE_SCROLL_INSENSITIVE,
+ ResultSet.CONCUR_READ_ONLY);
+ String locid_query = "select IPTOLOCID(\"" + metadata
+ + "\") as ANS";
+ resultSet = statement.executeQuery(locid_query);
+
+ if (resultSet == null)
+ throw new Exception("Invalid result set for metadata: "
+ + metadata + ". Query run was: " + locid_query);
+
+ resultSet.last();
+ int size = resultSet.getRow();
+
+ if (size == 0)
+ throw new Exception("No result returned for: " + metadata
+ + ". Query run was: " + locid_query);
+
+ resultSet.beforeFirst();
+ resultSet.next();
+
+ String locid = null;
+ locid = resultSet.getString("ANS");
+
+ if (locid == null)
+ throw new Exception("Invalid location id for: " + metadata
+ + ". Query run was: " + locid_query);
+
+ String geo_query = "select * from location where locID = " + locid
+ + ";";
+ resultSet = statement.executeQuery(geo_query);
+
+ if (resultSet == null)
+ throw new Exception(
+ "Invalid result set for metadata and locid: "
+ + metadata + ", " + locid + ". Query run was: "
+ + geo_query);
+
+ resultSet.last();
+ size = resultSet.getRow();
+
+ if (size == 0)
+ throw new Exception(
+ "No result id returned for metadata and locid: "
+ + metadata + ", " + locid + ". Query run was: "
+ + geo_query);
+
+ resultSet.beforeFirst();
+ resultSet.next();
+
+ JSONObject jo = new JSONObject();
+ jo.put("locID", resultSet.getString("locID"));
+ jo.put("country", resultSet.getString("country"));
+ jo.put("city", resultSet.getString("city"));
+ jo.put("postalCode", resultSet.getString("postalCode"));
+ jo.put("latitude", resultSet.getString("latitude"));
+ jo.put("longitude", resultSet.getString("longitude"));
+ jo.put("dmaCode", resultSet.getString("dmaCode"));
+ jo.put("locID", resultSet.getString("locID"));
+
+ jo.put("location_point", jo.get("longitude") + "," + jo.get("latitude"));
+
+ _LOG.debug("Returning enrichment: " + jo);
+
+ return jo;
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ _LOG.error("Enrichment failure: " + e);
+ return new JSONObject();
+ }
+ }
+
+ @Override
+ public boolean initializeAdapter() {
+
+ _LOG.info("[Metron] Initializing MysqlAdapter....");
+
+ try {
+
+ Class.forName("com.mysql.jdbc.Driver");
+ connection = DriverManager.getConnection("jdbc:mysql://" + _ip
+ + "/" + _tablename + "?user=" + _username + "&password="
+ + _password);
+
+ connection.setReadOnly(true);
+
+ if (!connection.isValid(0))
+ throw new Exception("Invalid connection string....");
+
+ _LOG.info("[Metron] Set JDBC connection....");
+
+ return true;
+ } catch (Exception e) {
+ e.printStackTrace();
+ _LOG.error("[Metron] JDBC connection failed....");
+
+ return false;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java
new file mode 100644
index 0000000..e3c878d
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/host/AbstractHostAdapter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apache.metron.enrichment.adapters.host;
+
+import java.io.Serializable;
+
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.apache.metron.enrichment.interfaces.EnrichmentAdapter;
+
+public abstract class AbstractHostAdapter implements EnrichmentAdapter,Serializable{
+
+ /**
+ * Adapter to attach reputation information to the telemetry message
+ */
+ private static final long serialVersionUID = 8280523289446309728L;
+ protected static final Logger LOG = LoggerFactory
+ .getLogger(AbstractHostAdapter.class);
+
+ abstract public boolean initializeAdapter();
+ abstract public JSONObject enrich(String metadata);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java
new file mode 100644
index 0000000..e5efdce
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/host/HostFromPropertiesFileAdapter.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apache.metron.enrichment.adapters.host;
+
+import java.util.Map;
+
+import org.json.simple.JSONObject;
+
+@SuppressWarnings("serial")
+public class HostFromPropertiesFileAdapter extends AbstractHostAdapter {
+
+ Map<String, JSONObject> _known_hosts;
+
+ public HostFromPropertiesFileAdapter(Map<String, JSONObject> known_hosts)
+ {
+ _known_hosts = known_hosts;
+ }
+
+ @Override
+ public boolean initializeAdapter()
+ {
+
+ if(_known_hosts.size() > 0)
+ return true;
+ else
+ return false;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public JSONObject enrich(String metadata) {
+
+
+ if(!_known_hosts.containsKey(metadata))
+ return new JSONObject();
+
+ JSONObject enrichment = new JSONObject();
+ enrichment.put("known_info", (JSONObject) _known_hosts.get(metadata));
+ return enrichment;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/threat/AbstractThreatAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/threat/AbstractThreatAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/threat/AbstractThreatAdapter.java
new file mode 100644
index 0000000..b12cbb7
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/threat/AbstractThreatAdapter.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apache.metron.enrichment.adapters.threat;
+
+import java.io.Serializable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.apache.metron.enrichment.interfaces.EnrichmentAdapter;
+
+public abstract class AbstractThreatAdapter implements EnrichmentAdapter,Serializable{
+
+
+ private static final long serialVersionUID = 1524030932856141771L;
+ protected static final Logger LOG = LoggerFactory
+ .getLogger(AbstractThreatAdapter.class);
+
+ abstract public boolean initializeAdapter();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/threat/ThreatHbaseAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/threat/ThreatHbaseAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/threat/ThreatHbaseAdapter.java
new file mode 100644
index 0000000..c08b87b
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/threat/ThreatHbaseAdapter.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apache.metron.enrichment.adapters.threat;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.log4j.Logger;
+
+@SuppressWarnings("unchecked")
+public class ThreatHbaseAdapter extends AbstractThreatAdapter {
+
+ private static final long serialVersionUID = 1L;
+ private String _tableName;
+ private HTableInterface table;
+ private String _quorum;
+ private String _port;
+
+ public ThreatHbaseAdapter(String quorum, String port, String tableName) {
+ _quorum = quorum;
+ _port = port;
+ _tableName = tableName;
+ }
+
+ /** The LOGGER. */
+ private static final Logger LOGGER = Logger
+ .getLogger(ThreatHbaseAdapter.class);
+
+ public JSONObject enrich(String metadata) {
+
+ JSONObject output = new JSONObject();
+ LOGGER.debug("=======Looking Up For:" + metadata);
+ output.putAll(getThreatObject(metadata));
+
+ return output;
+ }
+
+ @SuppressWarnings({ "rawtypes", "deprecation" })
+ protected Map getThreatObject(String key) {
+
+ LOGGER.debug("=======Pinging HBase For:" + key);
+
+ Get get = new Get(Bytes.toBytes(key));
+ Result rs;
+ Map output = new HashMap();
+
+ try {
+ rs = table.get(get);
+
+ if (!rs.isEmpty()) {
+ byte[] source_family = Bytes.toBytes("source");
+ JSONParser parser = new JSONParser();
+
+ Map<byte[], byte[]> sourceFamilyMap = rs.getFamilyMap(source_family);
+
+ for (Map.Entry<byte[], byte[]> entry : sourceFamilyMap.entrySet()) {
+ String k = Bytes.toString(entry.getKey());
+ LOGGER.debug("=======Found intel from source: " + k);
+ output.put(k,parser.parse(Bytes.toString(entry.getValue())));
+ }
+ }
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (ParseException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ return output;
+ }
+
+ @Override
+ public boolean initializeAdapter() {
+
+ // Initialize HBase Table
+ Configuration conf = null;
+ conf = HBaseConfiguration.create();
+ conf.set("hbase.zookeeper.quorum", _quorum);
+ conf.set("hbase.zookeeper.property.clientPort", _port);
+
+ try {
+ LOGGER.debug("=======Connecting to HBASE===========");
+ LOGGER.debug("=======ZOOKEEPER = "
+ + conf.get("hbase.zookeeper.quorum"));
+ HConnection connection = HConnectionManager.createConnection(conf);
+ table = connection.getTable(_tableName);
+ return true;
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ LOGGER.debug("=======Unable to Connect to HBASE===========");
+ e.printStackTrace();
+ }
+
+ return false;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/7c5661ea/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/whois/AbstractWhoisAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/whois/AbstractWhoisAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/whois/AbstractWhoisAdapter.java
new file mode 100644
index 0000000..89b2e55
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/enrichment/adapters/whois/AbstractWhoisAdapter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apache.metron.enrichment.adapters.whois;
+
+import java.io.Serializable;
+
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.apache.metron.enrichment.interfaces.EnrichmentAdapter;
+
+public abstract class AbstractWhoisAdapter implements EnrichmentAdapter,Serializable{
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 8280523289446309728L;
+ protected static final Logger LOG = LoggerFactory
+ .getLogger(AbstractWhoisAdapter.class);
+
+ abstract public boolean initializeAdapter();
+ abstract public JSONObject enrich(String metadata);
+
+}