You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/05/10 15:51:42 UTC
incubator-metron git commit: METRON-140 handle empty line in
CSVExtractor (mmiklavc via cestella) closes apache/incubator-metron#109
Repository: incubator-metron
Updated Branches:
refs/heads/master 4ddcf24dd -> 743f37b28
METRON-140 handle empty line in CSVExtractor (mmiklavc via cestella) closes apache/incubator-metron#109
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/743f37b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/743f37b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/743f37b2
Branch: refs/heads/master
Commit: 743f37b283efd1b46a98521de26648fcdba08474
Parents: 4ddcf24
Author: mmiklavc <mi...@clevelandflash.com>
Authored: Tue May 10 11:51:30 2016 -0400
Committer: cstella <ce...@gmail.com>
Committed: Tue May 10 11:51:30 2016 -0400
----------------------------------------------------------------------
.../dataloads/extractor/csv/CSVExtractor.java | 13 +++++++++++--
.../dataloads/extractor/csv/CSVExtractorTest.java | 18 ++++++++++++------
2 files changed, 23 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/743f37b2/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java
index a0f6b4d..28c3ece 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java
@@ -28,6 +28,8 @@ import org.apache.metron.enrichment.lookup.LookupKey;
import java.io.IOException;
import java.util.*;
+import static org.apache.commons.lang3.StringUtils.isEmpty;
+
public class CSVExtractor implements Extractor {
public static final String COLUMNS_KEY="columns";
public static final String INDICATOR_COLUMN_KEY="indicator_column";
@@ -68,8 +70,7 @@ public class CSVExtractor implements Extractor {
}
@Override
public Iterable<LookupKV> extract(String line) throws IOException {
- if(line.trim().startsWith("#")) {
- //comment
+ if(ignore(line)) {
return Collections.emptyList();
}
String[] tokens = parser.parseLine(line);
@@ -82,6 +83,14 @@ public class CSVExtractor implements Extractor {
return Arrays.asList(new LookupKV(key, converter.toValue(values)));
}
+ private boolean ignore(String line) {
+ if(null == line) {
+ return true;
+ }
+ String trimmedLine = line.trim();
+ return trimmedLine.startsWith("#") || isEmpty(trimmedLine);
+ }
+
private String getType(String[] tokens) {
if(type == null) {
return tokens[typeColumn];
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/743f37b2/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java
index 73f5bd9..4e482ad 100644
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java
@@ -23,16 +23,10 @@ import org.apache.metron.dataloads.extractor.ExtractorHandler;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
import org.apache.metron.enrichment.lookup.LookupKV;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
public class CSVExtractorTest {
@@ -91,5 +85,17 @@ public class CSVExtractorTest {
Iterable<LookupKV> results = handler.getExtractor().extract("#google.com,1.0,foo");
Assert.assertEquals(0, Iterables.size(results));
}
+ {
+ Iterable<LookupKV> results = handler.getExtractor().extract("");
+ Assert.assertEquals(0, Iterables.size(results));
+ }
+ {
+ Iterable<LookupKV> results = handler.getExtractor().extract(" ");
+ Assert.assertEquals(0, Iterables.size(results));
+ }
+ {
+ Iterable<LookupKV> results = handler.getExtractor().extract(null);
+ Assert.assertEquals(0, Iterables.size(results));
+ }
}
}