You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2017/02/24 21:00:03 UTC
incubator-metron git commit: METRON-733: Remove Geo database from
ParserBolt (justinleet via cestella) closes apache/incubator-metron#461
Repository: incubator-metron
Updated Branches:
refs/heads/master 84d347195 -> 898d2360c
METRON-733: Remove Geo database from ParserBolt (justinleet via cestella) closes apache/incubator-metron#461
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/898d2360
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/898d2360
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/898d2360
Branch: refs/heads/master
Commit: 898d2360c6344eb88698f17974e7a17760b316a3
Parents: 84d3471
Author: justinleet <ju...@gmail.com>
Authored: Fri Feb 24 15:59:55 2017 -0500
Committer: cstella <ce...@gmail.com>
Committed: Fri Feb 24 15:59:55 2017 -0500
----------------------------------------------------------------------
.../apache/metron/parsers/bolt/ParserBolt.java | 3 ---
.../metron/parsers/bolt/ParserBoltTest.java | 27 -------------------
.../integration/WriterBoltIntegrationTest.java | 1 -
.../resources/GeoLite/GeoIP2-City-Test.mmdb.gz | Bin 10011 -> 0 bytes
4 files changed, 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/898d2360/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
index 416d3ed..abbd9d8 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.parsers.bolt;
-import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase;
import org.apache.commons.lang3.StringUtils;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -73,8 +72,6 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable {
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
super.prepare(stormConf, context, collector);
- String hdfsFile = (String) getConfigurations().getGlobalConfig().get(GeoLiteDatabase.GEO_HDFS_FILE);
- GeoLiteDatabase.INSTANCE.update(hdfsFile);
this.collector = collector;
initializeStellar();
if(getSensorParserConfig() != null && filter == null) {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/898d2360/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
index de98a64..6b00998 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
@@ -19,7 +19,6 @@ package org.apache.metron.parsers.bolt;
import org.apache.metron.common.configuration.*;
-import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase;
import org.apache.metron.test.utils.UnitTestHelper;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
@@ -139,8 +138,6 @@ public class ParserBoltTest extends BaseBoltTest {
};
- buildGlobalConfig(parserBolt);
-
parserBolt.setCuratorFramework(client);
parserBolt.setTreeCache(cache);
parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
@@ -179,8 +176,6 @@ public class ParserBoltTest extends BaseBoltTest {
};
- buildGlobalConfig(parserBolt);
-
parserBolt.setCuratorFramework(client);
parserBolt.setTreeCache(cache);
parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
@@ -238,8 +233,6 @@ public void testImplicitBatchOfOne() throws Exception {
}
};
- buildGlobalConfig(parserBolt);
-
parserBolt.setCuratorFramework(client);
parserBolt.setTreeCache(cache);
parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
@@ -286,8 +279,6 @@ public void testImplicitBatchOfOne() throws Exception {
}
};
- buildGlobalConfig(parserBolt);
-
parserBolt.setCuratorFramework(client);
parserBolt.setTreeCache(cache);
parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
@@ -325,8 +316,6 @@ public void testImplicitBatchOfOne() throws Exception {
}
};
- buildGlobalConfig(parserBolt);
-
parserBolt.setCuratorFramework(client);
parserBolt.setTreeCache(cache);
parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
@@ -396,8 +385,6 @@ public void testImplicitBatchOfOne() throws Exception {
}
};
- buildGlobalConfig(parserBolt);
-
parserBolt.setCuratorFramework(client);
parserBolt.setTreeCache(cache);
parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
@@ -432,8 +419,6 @@ public void testImplicitBatchOfOne() throws Exception {
}
};
- buildGlobalConfig(parserBolt);
-
parserBolt.setCuratorFramework(client);
parserBolt.setTreeCache(cache);
parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
@@ -473,8 +458,6 @@ public void testImplicitBatchOfOne() throws Exception {
}
};
- buildGlobalConfig(parserBolt);
-
parserBolt.setCuratorFramework(client);
parserBolt.setTreeCache(cache);
parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
@@ -524,8 +507,6 @@ public void testImplicitBatchOfOne() throws Exception {
}
};
- buildGlobalConfig(parserBolt);
-
parserBolt.setCuratorFramework(client);
parserBolt.setTreeCache(cache);
parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
@@ -550,14 +531,6 @@ public void testImplicitBatchOfOne() throws Exception {
}
- protected void buildGlobalConfig(ParserBolt parserBolt) {
- HashMap<String, Object> globalConfig = new HashMap<>();
- String baseDir = UnitTestHelper.findDir("GeoLite");
- File geoHdfsFile = new File(new File(baseDir), "GeoIP2-City-Test.mmdb.gz");
- globalConfig.put(GeoLiteDatabase.GEO_HDFS_FILE, geoHdfsFile.getAbsolutePath());
- parserBolt.getConfigurations().updateGlobalConfig(globalConfig);
- }
-
private static void writeNonBatch(OutputCollector collector, ParserBolt bolt, Tuple t) {
bolt.execute(t);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/898d2360/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
index eaeaeb5..f37b1fc 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
@@ -61,7 +61,6 @@ public class WriterBoltIntegrationTest extends BaseIntegrationTest {
}
/**
{
- "geo.hdfs.file": "src/test/resources/GeoLite/GeoIP2-City-Test.mmdb.gz",
"fieldValidations" : [
{
"validation" : "org.apache.metron.writers.integration.WriterBoltIntegrationTest$MockValidator"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/898d2360/metron-platform/metron-parsers/src/test/resources/GeoLite/GeoIP2-City-Test.mmdb.gz
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/resources/GeoLite/GeoIP2-City-Test.mmdb.gz b/metron-platform/metron-parsers/src/test/resources/GeoLite/GeoIP2-City-Test.mmdb.gz
deleted file mode 100644
index 406c656..0000000
Binary files a/metron-platform/metron-parsers/src/test/resources/GeoLite/GeoIP2-City-Test.mmdb.gz and /dev/null differ