You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2016/04/28 17:10:26 UTC
incubator-metron git commit: METRON-121 Dataload Unit Tests (franklu1
via merrimanr) closes apache/incubator-metron#83
Repository: incubator-metron
Updated Branches:
refs/heads/master 29c5a039c -> 6a8772c3a
METRON-121 Dataload Unit Tests (franklu1 via merrimanr) closes apache/incubator-metron#83
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/6a8772c3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/6a8772c3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/6a8772c3
Branch: refs/heads/master
Commit: 6a8772c3a350dfbf54ba4249521d58d9339bb555
Parents: 29c5a03
Author: franklu1 <yl...@hortonworks.com>
Authored: Thu Apr 28 10:09:24 2016 -0500
Committer: rmerriman <rm...@hortonworks.com>
Committed: Thu Apr 28 10:09:24 2016 -0500
----------------------------------------------------------------------
.../dataloads/bulk/LeastRecentlyUsedPruner.java | 4 +-
.../dataloads/bulk/ThreatIntelBulkLoader.java | 3 +-
.../dataloads/extractor/csv/CSVExtractor.java | 23 +++
.../dataloads/nonbulk/taxii/TaxiiLoader.java | 2 +-
.../extractor/csv/CSVExtractorTest.java | 22 +++
.../hbase/mr/BulkLoadMapperIntegrationTest.java | 37 ++++-
.../LeastRecentlyUsedPrunerIntegrationTest.java | 36 ++++-
.../SimpleEnrichmentFlatFileLoaderTest.java | 146 +++++++++++++++++++
.../nonbulk/taxii/TaxiiIntegrationTest.java | 26 ++++
9 files changed, 287 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6a8772c3/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/LeastRecentlyUsedPruner.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/LeastRecentlyUsedPruner.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/LeastRecentlyUsedPruner.java
index 7acc96c..2f489e0 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/LeastRecentlyUsedPruner.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/LeastRecentlyUsedPruner.java
@@ -37,7 +37,7 @@ import java.util.Date;
public class LeastRecentlyUsedPruner {
private static abstract class OptionHandler implements Function<String, Option> {}
- private enum BulkLoadOptions {
+ public enum BulkLoadOptions {
HELP("h", new OptionHandler() {
@Nullable
@@ -74,7 +74,7 @@ public class LeastRecentlyUsedPruner {
return o;
}
})
- ,AS_OF_TIME_FORMAT("t", new OptionHandler() {
+ ,AS_OF_TIME_FORMAT("v", new OptionHandler() {
@Nullable
@Override
public Option apply(@Nullable String s) {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6a8772c3/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
index 1cc591b..2e63ef2 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
@@ -46,7 +46,7 @@ import java.util.Date;
public class ThreatIntelBulkLoader {
private static abstract class OptionHandler implements Function<String, Option> {}
- private enum BulkLoadOptions {
+ public enum BulkLoadOptions {
HELP("h", new OptionHandler() {
@Nullable
@@ -184,6 +184,7 @@ public class ThreatIntelBulkLoader {
return ret;
}
}
+
private static long getTimestamp(CommandLine cli) throws java.text.ParseException {
if(BulkLoadOptions.AS_OF_TIME.has(cli)) {
if(!BulkLoadOptions.AS_OF_TIME_FORMAT.has(cli)) {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6a8772c3/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java
index 1fce0fc..a0f6b4d 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java
@@ -43,6 +43,29 @@ public class CSVExtractor implements Extractor {
private CSVParser parser;
private LookupConverter converter = LookupConverters.ENRICHMENT.getConverter();
+ public int getTypeColumn() {
+ return typeColumn;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public int getIndicatorColumn() {
+ return indicatorColumn;
+ }
+
+ public Map<String, Integer> getColumnMap() {
+ return columnMap;
+ }
+
+ public CSVParser getParser() {
+ return parser;
+ }
+
+ public LookupConverter getConverter() {
+ return converter;
+ }
@Override
public Iterable<LookupKV> extract(String line) throws IOException {
if(line.trim().startsWith("#")) {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6a8772c3/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
index b5385a9..df803c1 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java
@@ -42,7 +42,7 @@ import java.util.Timer;
public class TaxiiLoader {
private static abstract class OptionHandler implements Function<String, Option> {}
- private enum TaxiiOptions {
+ public enum TaxiiOptions {
HELP("h", new OptionHandler() {
@Nullable
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6a8772c3/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java
index 6cd82c7..73f5bd9 100644
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java
@@ -23,10 +23,16 @@ import org.apache.metron.dataloads.extractor.ExtractorHandler;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
import org.apache.metron.enrichment.lookup.LookupKV;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
public class CSVExtractorTest {
@@ -47,6 +53,22 @@ public class CSVExtractorTest {
@Multiline
static String testCSVConfig;
+
+ @Test
+ public void testInitialize() throws Exception {
+ CSVExtractor ex = new CSVExtractor();
+ ExtractorHandler handler = ExtractorHandler.load(testCSVConfig);
+ ex.initialize(handler.getConfig());
+
+ Assert.assertEquals(0, (int)ex.getColumnMap().get("host") );
+ Assert.assertEquals(2, (int)ex.getColumnMap().get("meta") );
+ Assert.assertEquals(0, ex.getTypeColumn() );
+ Assert.assertEquals(0, ex.getIndicatorColumn());
+ Assert.assertEquals("threat", ex.getType() );
+ Assert.assertEquals(',', ex.getParser().getSeparator());
+
+ }
+
@Test
public void testCSVExtractor() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6a8772c3/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperIntegrationTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperIntegrationTest.java
index 626c98e..9971fa6 100644
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperIntegrationTest.java
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperIntegrationTest.java
@@ -18,20 +18,25 @@
package org.apache.metron.dataloads.hbase.mr;
import org.adrianwalker.multilinestring.Multiline;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.metron.dataloads.bulk.ThreatIntelBulkLoader;
import org.apache.metron.enrichment.converter.EnrichmentConverter;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
import org.apache.metron.enrichment.lookup.LookupKV;
+import org.apache.metron.dataloads.nonbulk.flatfile.SimpleEnrichmentFlatFileLoader;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -48,9 +53,17 @@ public class BulkLoadMapperIntegrationTest {
/** The test table. */
private HTable testTable;
- String tableName = "malicious_domains";
- String cf = "cf";
- Configuration config = null;
+ private String tableName = "malicious_domains";
+ private String cf = "cf";
+ private String csvFile="input.csv";
+ private String extractorJson = "extractor.json";
+ private String enrichmentJson = "enrichment_config.json";
+ private String asOf = "04/15/2016";
+ private String asOfFormat = "georgia";
+ private String convertClass = "threadIntel.class";
+ private Configuration config = null;
+
+
@Before
public void setup() throws Exception {
Map.Entry<HBaseTestingUtility, Configuration> kv = HBaseUtil.INSTANCE.create(true);
@@ -81,6 +94,24 @@ public class BulkLoadMapperIntegrationTest {
private static String extractorConfig;
@Test
+ public void testCommandLine() throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+
+ String[] argv = {"-f cf", "-t malicious_domains", "-e extractor.json", "-n enrichment_config.json", "-a 04/15/2016", "-i input.csv", "-z georgia", "-c threadIntel.class"};
+ String[] otherArgs = new GenericOptionsParser(conf, argv).getRemainingArgs();
+
+ CommandLine cli = ThreatIntelBulkLoader.BulkLoadOptions.parse(new PosixParser(), otherArgs);
+ Assert.assertEquals(extractorJson,ThreatIntelBulkLoader.BulkLoadOptions.EXTRACTOR_CONFIG.get(cli).trim());
+ Assert.assertEquals(cf, ThreatIntelBulkLoader.BulkLoadOptions.COLUMN_FAMILY.get(cli).trim());
+ Assert.assertEquals(tableName,ThreatIntelBulkLoader.BulkLoadOptions.TABLE.get(cli).trim());
+ Assert.assertEquals(enrichmentJson,ThreatIntelBulkLoader.BulkLoadOptions.ENRICHMENT_CONFIG.get(cli).trim());
+ Assert.assertEquals(csvFile,ThreatIntelBulkLoader.BulkLoadOptions.INPUT_DATA.get(cli).trim());
+ Assert.assertEquals(asOf, ThreatIntelBulkLoader.BulkLoadOptions.AS_OF_TIME.get(cli).trim());
+ Assert.assertEquals(asOfFormat, ThreatIntelBulkLoader.BulkLoadOptions.AS_OF_TIME_FORMAT.get(cli).trim());
+ Assert.assertEquals(convertClass, ThreatIntelBulkLoader.BulkLoadOptions.CONVERTER.get(cli).trim());
+ }
+
+ @Test
public void test() throws IOException, ClassNotFoundException, InterruptedException {
Assert.assertNotNull(testTable);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6a8772c3/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/LeastRecentlyUsedPrunerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/LeastRecentlyUsedPrunerIntegrationTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/LeastRecentlyUsedPrunerIntegrationTest.java
index 65befe3..42ec575 100644
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/LeastRecentlyUsedPrunerIntegrationTest.java
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/hbase/mr/LeastRecentlyUsedPrunerIntegrationTest.java
@@ -18,11 +18,15 @@
package org.apache.metron.dataloads.hbase.mr;
import com.google.common.collect.Iterables;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.metron.dataloads.bulk.LeastRecentlyUsedPruner;
import org.apache.metron.enrichment.converter.EnrichmentConverter;
import org.apache.metron.enrichment.converter.EnrichmentKey;
@@ -31,6 +35,8 @@ import org.apache.metron.enrichment.lookup.EnrichmentLookup;
import org.apache.metron.enrichment.lookup.LookupKey;
import org.apache.metron.enrichment.lookup.accesstracker.BloomAccessTracker;
import org.apache.metron.enrichment.lookup.accesstracker.PersistentAccessTracker;
+import org.apache.metron.dataloads.bulk.ThreatIntelBulkLoader;
+import org.apache.metron.dataloads.nonbulk.taxii.TaxiiLoader;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -48,11 +54,14 @@ public class LeastRecentlyUsedPrunerIntegrationTest {
/** The test table. */
private HTable testTable;
private HTable atTable;
- String tableName = "malicious_domains";
- String cf = "cf";
- String atTableName = "access_trackers";
- String atCF= "cf";
- Configuration config = null;
+ private String tableName = "malicious_domains";
+ private String cf = "cf";
+ private String atTableName = "access_trackers";
+ private String atCF= "cf";
+ private String beginTime = "04/14/2016 12:00:00";
+ private String timeFormat = "georgia";
+ private Configuration config = null;
+
@Before
public void setup() throws Exception {
Map.Entry<HBaseTestingUtility, Configuration> kv = HBaseUtil.INSTANCE.create(true);
@@ -72,6 +81,23 @@ public class LeastRecentlyUsedPrunerIntegrationTest {
}
return keys;
}
+
+ @Test
+ public void testCommandLine() throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+
+ String[] argv = {"-a 04/14/2016 12:00:00", "-f cf", "-t malicious_domains", "-u access_trackers", "-v georgia", "-z cf"};
+ String[] otherArgs = new GenericOptionsParser(conf, argv).getRemainingArgs();
+
+ CommandLine cli = LeastRecentlyUsedPruner.BulkLoadOptions.parse(new PosixParser(), otherArgs);
+ Assert.assertEquals(cf, LeastRecentlyUsedPruner.BulkLoadOptions.COLUMN_FAMILY.get(cli).trim());
+ Assert.assertEquals(tableName,LeastRecentlyUsedPruner.BulkLoadOptions.TABLE.get(cli).trim());
+ Assert.assertEquals(atTableName,LeastRecentlyUsedPruner.BulkLoadOptions.ACCESS_TABLE.get(cli).trim());
+ Assert.assertEquals(atCF,LeastRecentlyUsedPruner.BulkLoadOptions.ACCESS_COLUMN_FAMILY.get(cli).trim());
+ Assert.assertEquals(beginTime, LeastRecentlyUsedPruner.BulkLoadOptions.AS_OF_TIME.get(cli).trim());
+ Assert.assertEquals(timeFormat, LeastRecentlyUsedPruner.BulkLoadOptions.AS_OF_TIME_FORMAT.get(cli).trim());
+ }
+
@Test
public void test() throws Exception {
long ts = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6a8772c3/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoaderTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoaderTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoaderTest.java
new file mode 100644
index 0000000..b4891aa
--- /dev/null
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/flatfile/SimpleEnrichmentFlatFileLoaderTest.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.dataloads.nonbulk.flatfile;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.log4j.PropertyConfigurator;
+import org.apache.metron.dataloads.bulk.ThreatIntelBulkLoader;
+import org.apache.metron.dataloads.extractor.Extractor;
+import org.apache.metron.dataloads.extractor.ExtractorHandler;
+import org.apache.metron.dataloads.extractor.inputformat.WholeFileFormat;
+import org.apache.metron.dataloads.nonbulk.flatfile.SimpleEnrichmentFlatFileLoader;
+import org.apache.metron.dataloads.hbase.mr.HBaseUtil;
+import org.apache.metron.enrichment.converter.HbaseConverter;
+import org.apache.metron.enrichment.converter.EnrichmentConverter;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.lookup.LookupKV;
+import org.apache.metron.common.utils.JSONUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class SimpleEnrichmentFlatFileLoaderTest {
+
+ private HBaseTestingUtility testUtil;
+
+ /** The test table. */
+ private HTable testTable;
+ private String tableName = "enrichment";
+ private String cf = "cf";
+ private String csvFile="input.csv";
+ private String extractorJson = "extractor.json";
+ private String enrichmentJson = "enrichment_config.json";
+ private String log4jProperty = "log4j";
+
+ Configuration config = null;
+ /**
+ {
+ "config" : {
+ "columns" : {
+ "host" : 0,
+ "meta" : 2
+ },
+ "indicator_column" : "host",
+ "separator" : ",",
+ "type" : "enrichment"
+ },
+ "extractor" : "CSV"
+ }
+ */
+ @Multiline
+ private static String extractorConfig;
+
+ @Before
+ public void setup() throws Exception {
+ Map.Entry<HBaseTestingUtility, Configuration> kv = HBaseUtil.INSTANCE.create(true);
+ config = kv.getValue();
+ testUtil = kv.getKey();
+ testTable = testUtil.createTable(Bytes.toBytes(tableName), Bytes.toBytes(cf));
+ }
+
+ @After
+ public void teardown() throws Exception {
+ HBaseUtil.INSTANCE.teardown(testUtil);
+ }
+
+ @Test
+ public void testCommandLine() throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+
+ String[] argv = {"-c cf", "-t enrichment", "-e extractor.json", "-n enrichment_config.json", "-l log4j", "-i input.csv"};
+ String[] otherArgs = new GenericOptionsParser(conf, argv).getRemainingArgs();
+
+ CommandLine cli = SimpleEnrichmentFlatFileLoader.LoadOptions.parse(new PosixParser(), otherArgs);
+ Assert.assertEquals(extractorJson,SimpleEnrichmentFlatFileLoader.LoadOptions.EXTRACTOR_CONFIG.get(cli).trim());
+ Assert.assertEquals(cf, SimpleEnrichmentFlatFileLoader.LoadOptions.HBASE_CF.get(cli).trim());
+ Assert.assertEquals(tableName,SimpleEnrichmentFlatFileLoader.LoadOptions.HBASE_TABLE.get(cli).trim());
+ Assert.assertEquals(enrichmentJson,SimpleEnrichmentFlatFileLoader.LoadOptions.ENRICHMENT_CONFIG.get(cli).trim());
+ Assert.assertEquals(csvFile,SimpleEnrichmentFlatFileLoader.LoadOptions.INPUT.get(cli).trim());
+ Assert.assertEquals(log4jProperty, SimpleEnrichmentFlatFileLoader.LoadOptions.LOG4J_PROPERTIES.get(cli).trim());
+ }
+
+ @Test
+ public void test() throws Exception {
+
+ Assert.assertNotNull(testTable);
+ String contents = "google.com,1,foo";
+
+ EnrichmentConverter converter = new EnrichmentConverter();
+ ExtractorHandler handler = ExtractorHandler.load(extractorConfig);
+ Extractor e = handler.getExtractor();
+ File file = new File (contents);
+ SimpleEnrichmentFlatFileLoader loader = new SimpleEnrichmentFlatFileLoader();
+ testTable.put(loader.extract(contents, e, cf, converter));
+
+ ResultScanner scanner = testTable.getScanner(Bytes.toBytes(cf));
+ List<LookupKV<EnrichmentKey, EnrichmentValue>> results = new ArrayList<>();
+ for(Result r : scanner) {
+ results.add(converter.fromResult(r, cf));
+ }
+ Assert.assertEquals(1, results.size());
+ Assert.assertEquals(results.get(0).getKey().indicator, "google.com");
+ Assert.assertEquals(results.get(0).getKey().type, "enrichment");
+ Assert.assertEquals(results.get(0).getValue().getMetadata().size(), 2);
+ Assert.assertEquals(results.get(0).getValue().getMetadata().get("meta"), "foo");
+ Assert.assertEquals(results.get(0).getValue().getMetadata().get("host"), "google.com");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6a8772c3/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java
index f0d9178..d70425d 100644
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java
@@ -20,10 +20,13 @@ package org.apache.metron.dataloads.nonbulk.taxii;
import com.google.common.base.Splitter;
import org.adrianwalker.multilinestring.Multiline;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.metron.dataloads.extractor.stix.StixExtractor;
import org.apache.metron.enrichment.converter.EnrichmentConverter;
import org.apache.metron.enrichment.converter.EnrichmentKey;
@@ -65,6 +68,29 @@ public class TaxiiIntegrationTest {
@Multiline
static String taxiiConnectionConfig;
+ private String connectionConfig = "connection.json";
+ private String extractorJson = "extractor.json";
+ private String enrichmentJson = "enrichment_config.json";
+ private String log4jProperty = "log4j";
+ private String beginTime = "04/14/2016 12:00:00";
+ private String timeInteval = "10";
+
+ @Test
+ public void testCommandLine() throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+
+ String[] argv = {"-c connection.json", "-e extractor.json", "-n enrichment_config.json", "-l log4j", "-p 10", "-b 04/14/2016 12:00:00"};
+ String[] otherArgs = new GenericOptionsParser(conf, argv).getRemainingArgs();
+
+ CommandLine cli = TaxiiLoader.TaxiiOptions.parse(new PosixParser(), otherArgs);
+ Assert.assertEquals(extractorJson,TaxiiLoader.TaxiiOptions.EXTRACTOR_CONFIG.get(cli).trim());
+ Assert.assertEquals(connectionConfig, TaxiiLoader.TaxiiOptions.CONNECTION_CONFIG.get(cli).trim());
+ Assert.assertEquals(beginTime,TaxiiLoader.TaxiiOptions.BEGIN_TIME.get(cli).trim());
+ Assert.assertEquals(enrichmentJson,TaxiiLoader.TaxiiOptions.ENRICHMENT_CONFIG.get(cli).trim());
+ Assert.assertEquals(timeInteval,TaxiiLoader.TaxiiOptions.TIME_BETWEEN_POLLS.get(cli).trim());
+ Assert.assertEquals(log4jProperty, TaxiiLoader.TaxiiOptions.LOG4J_PROPERTIES.get(cli).trim());
+ }
+
@Test
public void testTaxii() throws Exception {