You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/05/31 21:00:28 UTC
[2/3] incubator-metron git commit: METRON-174 Storm consumption of
hbase enrichment reference data. This closes apache/incubator-metron#127
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java
index d9b7b38..bd019a0 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java
@@ -35,20 +35,20 @@ public class EnrichmentValue implements LookupValue {
public static final String VALUE_COLUMN_NAME = "v";
public static final byte[] VALUE_COLUMN_NAME_B = Bytes.toBytes(VALUE_COLUMN_NAME);
- private Map<String, String> metadata = null;
+ private Map<String, Object> metadata = null;
public EnrichmentValue()
{
}
- public EnrichmentValue(Map<String, String> metadata) {
+ public EnrichmentValue(Map<String, Object> metadata) {
this.metadata = metadata;
}
- public Map<String, String> getMetadata() {
+ public Map<String, Object> getMetadata() {
return metadata;
}
@@ -66,14 +66,14 @@ public class EnrichmentValue implements LookupValue {
}
}
}
- public Map<String, String> stringToValue(String s){
+ public Map<String, Object> stringToValue(String s){
try {
- return _mapper.get().readValue(s, new TypeReference<Map<String, String>>(){});
+ return _mapper.get().readValue(s, new TypeReference<Map<String, Object>>(){});
} catch (IOException e) {
throw new RuntimeException("Unable to convert string to metadata: " + s);
}
}
- public String valueToString(Map<String, String> value) {
+ public String valueToString(Map<String, Object> value) {
try {
return _mapper.get().writeValueAsString(value);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
new file mode 100644
index 0000000..f7e7aeb
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
@@ -0,0 +1,282 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.hbase;
+
+import backtype.storm.tuple.Tuple;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.common.utils.ReflectionUtils;
+import org.apache.metron.common.writer.AbstractWriter;
+import org.apache.metron.enrichment.converter.EnrichmentConverter;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.converter.HbaseConverter;
+import org.apache.metron.hbase.HTableProvider;
+import org.apache.metron.hbase.TableProvider;
+import org.json.simple.JSONObject;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public class SimpleHbaseEnrichmentWriter extends AbstractWriter implements BulkMessageWriter<JSONObject>, Serializable {
+ public enum Configurations {
+ HBASE_TABLE("shew.table")
+ ,HBASE_CF("shew.cf")
+ ,KEY_COLUMNS("shew.keyColumns")
+ ,KEY_DELIM("shew.keyDelim")
+ ,ENRICHMENT_TYPE("shew.enrichmentType")
+ ,VALUE_COLUMNS("shew.valueColumns")
+ ,HBASE_PROVIDER("shew.hbaseProvider")
+ ;
+ String key;
+ Configurations(String key) {
+ this.key = key;
+ }
+ public String getKey() {
+ return key;
+ }
+ public Object get(Map<String, Object> config) {
+ return config.get(key);
+ }
+ public <T> T getAndConvert(Map<String, Object> config, Class<T> clazz) {
+ Object o = get(config);
+ if(o != null) {
+ return ConversionUtils.convert(o, clazz);
+ }
+ return null;
+ }
+ }
+ public static class KeyTransformer {
+ List<String> keys = new ArrayList<>();
+ Set<String> keySet;
+ private String delim = ":";
+ public KeyTransformer(String key) {
+ this(key, null);
+ }
+ public KeyTransformer(String key, String delim) {
+ keys.add(key);
+ keySet = new HashSet<>(this.keys);
+ this.delim = delim == null?this.delim:delim;
+ }
+ public KeyTransformer(Iterable<String> keys) {
+ this(keys, null);
+ }
+ public KeyTransformer(Iterable<String> keys, String delim) {
+ Iterables.addAll(this.keys, keys);
+ keySet = new HashSet<>(this.keys);
+ this.delim = delim == null?this.delim:delim;
+ }
+
+ public String transform(final JSONObject message) {
+ return
+ keys.stream().map( x -> {
+ Object o = message.get(x);
+ return o == null?"":o.toString();
+ }).collect(Collectors.joining(delim));
+ }
+ }
+ private transient EnrichmentConverter converter;
+ private String tableName;
+ private String cf;
+ private HTableInterface table;
+ private TableProvider provider;
+ private Map.Entry<Object, KeyTransformer> keyTransformer;
+
+ public SimpleHbaseEnrichmentWriter() {
+ }
+
+ @Override
+ public void configure(String sensorName, WriterConfiguration configuration) {
+ String hbaseProviderImpl = Configurations.HBASE_PROVIDER.getAndConvert(configuration.getSensorConfig(sensorName),String.class);
+ if(hbaseProviderImpl != null) {
+ provider = ReflectionUtils.createInstance(hbaseProviderImpl);
+ }
+ if(converter == null) {
+ converter = new EnrichmentConverter();
+ }
+ }
+
+ @Override
+ public void init(Map stormConf, WriterConfiguration configuration) throws Exception {
+ if(converter == null) {
+ converter = new EnrichmentConverter();
+ }
+ }
+
+ protected synchronized TableProvider getProvider() {
+ if(provider == null) {
+
+ provider = new HTableProvider();
+ }
+ return provider;
+ }
+
+ public HTableInterface getTable(String tableName, String cf) throws IOException {
+ synchronized(this) {
+ boolean isInitial = this.tableName == null || this.cf == null;
+ boolean isValid = tableName != null && cf != null;
+
+ if( isInitial || (isValid && (!this.tableName.equals(tableName) || !this.cf.equals(cf)) )
+ )
+ {
+ Configuration conf = HBaseConfiguration.create();
+ //new table connection
+ if(table != null) {
+ table.close();
+ }
+ table = getProvider().getTable(conf, tableName);
+ this.tableName = tableName;
+ this.cf = cf;
+ }
+ return table;
+ }
+ }
+
+ public HTableInterface getTable(Map<String, Object> config) throws IOException {
+ return getTable(Configurations.HBASE_TABLE.getAndConvert(config, String.class)
+ ,Configurations.HBASE_CF.getAndConvert(config, String.class)
+ );
+
+ }
+
+
+ private List<String> getColumns(Object keyColumnsObj, boolean allowNull) {
+ Object o = keyColumnsObj;
+ if(allowNull && keyColumnsObj == null) {
+ return Collections.emptyList();
+ }
+ if(o instanceof String) {
+ return ImmutableList.of(o.toString());
+ }
+ else if (o instanceof List) {
+ List<String> keyCols = new ArrayList<>();
+ for(Object key : (List)o) {
+ keyCols.add(key.toString());
+ }
+ return keyCols;
+ }
+ else {
+ throw new RuntimeException("Unable to get columns: " + o);
+ }
+ }
+
+ private KeyTransformer getTransformer(Map<String, Object> config) {
+ Object o = Configurations.KEY_COLUMNS.get(config);
+ KeyTransformer transformer = null;
+ if(keyTransformer != null && keyTransformer.getKey() == o) {
+ return keyTransformer.getValue();
+ }
+ else {
+ List<String> keys = getColumns(o, false);
+ Object delimObj = Configurations.KEY_DELIM.get(config);
+ String delim = (delimObj == null || !(delimObj instanceof String))?null:delimObj.toString();
+ transformer = new KeyTransformer(keys, delim);
+ keyTransformer = new AbstractMap.SimpleEntry<>(o, transformer);
+ return transformer;
+ }
+ }
+
+
+ private EnrichmentValue getValue( JSONObject message
+ , Set<String> keyColumns
+ , Set<String> valueColumns
+ )
+ {
+ Map<String, Object> metadata = new HashMap<>();
+ if(valueColumns == null || valueColumns.isEmpty()) {
+ for (Object kv : message.entrySet()) {
+ Map.Entry<Object, Object> entry = (Map.Entry<Object, Object>) kv;
+ if (!keyColumns.contains(entry.getKey())) {
+ metadata.put(entry.getKey().toString(), entry.getValue());
+ }
+ }
+ return new EnrichmentValue(metadata);
+ }
+ else {
+ for (Object kv : message.entrySet()) {
+ Map.Entry<Object, Object> entry = (Map.Entry<Object, Object>) kv;
+ if (valueColumns.contains(entry.getKey())) {
+ metadata.put(entry.getKey().toString(), entry.getValue());
+ }
+ }
+ return new EnrichmentValue(metadata);
+ }
+ }
+
+ private EnrichmentKey getKey(JSONObject message, KeyTransformer transformer, String enrichmentType) {
+ if(enrichmentType != null) {
+ return new EnrichmentKey(enrichmentType, transformer.transform(message));
+ }
+ else {
+ return null;
+ }
+ }
+
+ @Override
+ public void write( String sensorType
+ , WriterConfiguration configurations
+ , Iterable<Tuple> tuples
+ , List<JSONObject> messages
+ ) throws Exception
+ {
+ Map<String, Object> sensorConfig = configurations.getSensorConfig(sensorType);
+ HTableInterface table = getTable(sensorConfig);
+ KeyTransformer transformer = getTransformer(sensorConfig);
+ Object enrichmentTypeObj = Configurations.ENRICHMENT_TYPE.get(sensorConfig);
+ String enrichmentType = enrichmentTypeObj == null?null:enrichmentTypeObj.toString();
+ Set<String> valueColumns = new HashSet<>(getColumns(Configurations.VALUE_COLUMNS.get(sensorConfig), true));
+ List<Put> puts = new ArrayList<>();
+ for(JSONObject message : messages) {
+ EnrichmentKey key = getKey(message, transformer, enrichmentType);
+ EnrichmentValue value = getValue(message, transformer.keySet, valueColumns);
+ if(key == null || value == null) {
+ continue;
+ }
+ Put put = converter.toPut(this.cf, key, value);
+ if(put != null) {
+ puts.add(put);
+ }
+ }
+ table.put(puts);
+ }
+
+ @Override
+ public void close() throws Exception {
+ synchronized(this) {
+ if(table != null) {
+ table.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
index d8f0e73..01f1245 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
@@ -19,6 +19,7 @@ package org.apache.metron.writer.hdfs;
import backtype.storm.tuple.Tuple;
import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.interfaces.BulkMessageWriter;
import org.apache.storm.hdfs.bolt.format.FileNameFormat;
import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
@@ -62,14 +63,15 @@ public class HdfsWriter implements BulkMessageWriter<JSONObject>, Serializable {
}
@Override
- public void init(Map stormConfig, EnrichmentConfigurations configurations) {
+ public void init(Map stormConfig, WriterConfiguration configurations) {
this.stormConfig = stormConfig;
}
+
@Override
public void write(String sourceType
- , EnrichmentConfigurations configurations
- , List<Tuple> tuples
+ , WriterConfiguration configurations
+ , Iterable<Tuple> tuples
, List<JSONObject> messages
) throws Exception
{
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
index 1a958df..bf72219 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
@@ -50,10 +50,10 @@ public class SimpleHBaseAdapterTest {
private EnrichmentLookup lookup;
private static final String PLAYFUL_CLASSIFICATION_TYPE = "playful_classification";
private static final String CF1_CLASSIFICATION_TYPE = "cf1";
- private static final Map<String, String> CF1_ENRICHMENT = new HashMap<String, String>() {{
+ private static final Map<String, Object> CF1_ENRICHMENT = new HashMap<String, Object>() {{
put("key", "value");
}};
- private static final Map<String, String> PLAYFUL_ENRICHMENT = new HashMap<String, String>() {{
+ private static final Map<String, Object> PLAYFUL_ENRICHMENT = new HashMap<String, Object>() {{
put("orientation", "north");
}};
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
index f420b01..46216b2 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
@@ -101,7 +101,7 @@ public class ThreatIntelAdapterTest {
final MockHTable trackerTable = (MockHTable) MockHTable.Provider.addToCache(atTableName, cf);
final MockHTable threatIntelTable = (MockHTable) MockHTable.Provider.addToCache(threatIntelTableName, cf);
EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {{
- add(new LookupKV<>(new EnrichmentKey("10.0.2.3", "10.0.2.3"), new EnrichmentValue(new HashMap<String, String>())));
+ add(new LookupKV<>(new EnrichmentKey("10.0.2.3", "10.0.2.3"), new EnrichmentValue(new HashMap<>())));
}});
BloomAccessTracker bat = new BloomAccessTracker(threatIntelTableName, 100, 0.03);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
index 61961a7..332ded0 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
@@ -22,6 +22,7 @@ import backtype.storm.tuple.Values;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.test.bolt.BaseEnrichmentBoltTest;
import org.apache.metron.common.configuration.Configurations;
import org.apache.metron.common.interfaces.BulkMessageWriter;
@@ -117,28 +118,28 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
bulkMessageWriterBolt.declareOutputFields(declarer);
verify(declarer, times(1)).declareStream(eq("error"), argThat(new FieldsMatcher("message")));
Map stormConf = new HashMap();
- doThrow(new Exception()).when(bulkMessageWriter).init(eq(stormConf), any(EnrichmentConfigurations.class));
+ doThrow(new Exception()).when(bulkMessageWriter).init(eq(stormConf), any(WriterConfiguration.class));
try {
bulkMessageWriterBolt.prepare(stormConf, topologyContext, outputCollector);
fail("A runtime exception should be thrown when bulkMessageWriter.init throws an exception");
} catch(RuntimeException e) {}
reset(bulkMessageWriter);
bulkMessageWriterBolt.prepare(stormConf, topologyContext, outputCollector);
- verify(bulkMessageWriter, times(1)).init(eq(stormConf), any(EnrichmentConfigurations.class));
+ verify(bulkMessageWriter, times(1)).init(eq(stormConf), any(WriterConfiguration.class));
tupleList = new ArrayList<>();
for(int i = 0; i < 4; i++) {
when(tuple.getValueByField("message")).thenReturn(messageList.get(i));
tupleList.add(tuple);
bulkMessageWriterBolt.execute(tuple);
- verify(bulkMessageWriter, times(0)).write(eq(sensorType), any(EnrichmentConfigurations.class), eq(tupleList), eq(messageList));
+ verify(bulkMessageWriter, times(0)).write(eq(sensorType), any(WriterConfiguration.class), eq(tupleList), eq(messageList));
}
when(tuple.getValueByField("message")).thenReturn(messageList.get(4));
tupleList.add(tuple);
bulkMessageWriterBolt.execute(tuple);
- verify(bulkMessageWriter, times(1)).write(eq(sensorType), any(EnrichmentConfigurations.class), eq(tupleList), argThat(new MessageListMatcher(messageList)));
+ verify(bulkMessageWriter, times(1)).write(eq(sensorType), any(WriterConfiguration.class), eq(tupleList), argThat(new MessageListMatcher(messageList)));
verify(outputCollector, times(5)).ack(tuple);
reset(outputCollector);
- doThrow(new Exception()).when(bulkMessageWriter).write(eq(sensorType), any(EnrichmentConfigurations.class), Matchers.anyListOf(Tuple.class), Matchers.anyListOf(JSONObject.class));
+ doThrow(new Exception()).when(bulkMessageWriter).write(eq(sensorType), any(WriterConfiguration.class), Matchers.anyListOf(Tuple.class), Matchers.anyListOf(JSONObject.class));
when(tuple.getValueByField("message")).thenReturn(messageList.get(0));
for(int i = 0; i < 5; i++) {
bulkMessageWriterBolt.execute(tuple);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/converter/EnrichmentConverterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/converter/EnrichmentConverterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/converter/EnrichmentConverterTest.java
index 20ec64c..e506c97 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/converter/EnrichmentConverterTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/converter/EnrichmentConverterTest.java
@@ -39,7 +39,7 @@ public class EnrichmentConverterTest {
public void testValueConversion() throws IOException {
EnrichmentConverter converter = new EnrichmentConverter();
EnrichmentKey k1 = new EnrichmentKey("type", "indicator");
- EnrichmentValue v1 = new EnrichmentValue(new HashMap<String, String>() {{
+ EnrichmentValue v1 = new EnrichmentValue(new HashMap<String, Object>() {{
put("k1", "v1");
put("k2", "v2");
}});
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java
index 1fd69b3..e4d48dc 100644
--- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java
+++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.hbase.HTableProvider;
import org.apache.metron.hbase.TableProvider;
import org.apache.metron.common.utils.ReflectionUtils;
@@ -62,7 +63,7 @@ public abstract class HBaseWriter implements MessageWriter<JSONObject>, Serializ
}
@Override
- public void write(String sourceType, Configurations configurations, Tuple tuple, JSONObject message) throws Exception {
+ public void write(String sourceType, WriterConfiguration configurations, Tuple tuple, JSONObject message) throws Exception {
Put put = new Put(getKey(tuple, message));
Map<String, byte[]> values = getValues(tuple, message);
for(String column: values.keySet()) {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
index 521ffdf..483f2b5 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
@@ -65,7 +65,7 @@ public abstract class EnrichmentIntegrationTest extends BaseIntegrationTest {
private static final String DST_IP = "ip_dst_addr";
private static final String MALICIOUS_IP_TYPE = "malicious_ip";
private static final String PLAYFUL_CLASSIFICATION_TYPE = "playful_classification";
- private static final Map<String, String> PLAYFUL_ENRICHMENT = new HashMap<String, String>() {{
+ private static final Map<String, Object> PLAYFUL_ENRICHMENT = new HashMap<String, Object>() {{
put("orientation", "north");
}};
protected String testSensorType = "test";
@@ -183,7 +183,7 @@ public abstract class EnrichmentIntegrationTest extends BaseIntegrationTest {
final MockHTable trackerTable = (MockHTable)MockHTable.Provider.addToCache(trackerHBaseTableName, cf);
final MockHTable threatIntelTable = (MockHTable)MockHTable.Provider.addToCache(threatIntelTableName, cf);
EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>(){{
- add(new LookupKV<>(new EnrichmentKey(MALICIOUS_IP_TYPE, "10.0.2.3"), new EnrichmentValue(new HashMap<String, String>())));
+ add(new LookupKV<>(new EnrichmentKey(MALICIOUS_IP_TYPE, "10.0.2.3"), new EnrichmentValue(new HashMap<>())));
}});
final MockHTable enrichmentTable = (MockHTable)MockHTable.Provider.addToCache(enrichmentsTableName, cf);
EnrichmentHelper.INSTANCE.load(enrichmentTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>(){{
@@ -206,7 +206,7 @@ public abstract class EnrichmentIntegrationTest extends BaseIntegrationTest {
.withComponent("config", configUploadComponent)
.withComponent("search", searchComponent)
.withComponent("storm", fluxComponent)
- .withMillisecondsBetweenAttempts(10000)
+ .withMillisecondsBetweenAttempts(15000)
.withNumRetries(10)
.build();
runner.start();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
index 0a088cb..865b017 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
@@ -23,6 +23,8 @@ import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.integration.InMemoryComponent;
import org.apache.metron.integration.UnableToStartException;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Properties;
public class ConfigUploadComponent implements InMemoryComponent {
@@ -31,7 +33,7 @@ public class ConfigUploadComponent implements InMemoryComponent {
private String globalConfigPath;
private String parserConfigsPath;
private String enrichmentConfigsPath;
-
+ private Map<String, SensorParserConfig> parserSensorConfigs = new HashMap<>();
public ConfigUploadComponent withTopologyProperties(Properties topologyProperties) {
this.topologyProperties = topologyProperties;
return this;
@@ -51,11 +53,27 @@ public class ConfigUploadComponent implements InMemoryComponent {
return this;
}
+ public ConfigUploadComponent withParserSensorConfig(String name, SensorParserConfig config) {
+ parserSensorConfigs.put(name, config);
+ return this;
+ }
+
@Override
public void start() throws UnableToStartException {
try {
- ConfigurationsUtils.uploadConfigsToZookeeper(globalConfigPath, parserConfigsPath, enrichmentConfigsPath, topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY));
+ ConfigurationsUtils.uploadConfigsToZookeeper( globalConfigPath
+ , parserConfigsPath
+ , enrichmentConfigsPath
+ , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
+ );
+ for(Map.Entry<String, SensorParserConfig> kv : parserSensorConfigs.entrySet()) {
+ ConfigurationsUtils.writeSensorParserConfigToZookeeper( kv.getKey()
+ , kv.getValue()
+ , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
+ );
+ }
+
} catch (Exception e) {
throw new UnableToStartException(e.getMessage(), e);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java
index f991fe2..99f916d 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java
@@ -136,6 +136,7 @@ public class KafkaWithZKComponent implements InMemoryComponent {
// setup Broker
Properties props = TestUtils.createBrokerConfig(0, brokerPort, true);
+ props.setProperty("zookeeper.connection.timeout.ms","1000000");
KafkaConfig config = new KafkaConfig(props);
Time mock = new MockTime();
kafkaServer = TestUtils.createServer(config, mock);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java
new file mode 100644
index 0000000..becadce
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.integration.mock;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.test.mock.MockHTable;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+public class MockTableProvider implements TableProvider, Serializable {
+ static MockHTable.Provider provider = new MockHTable.Provider();
+ @Override
+ public HTableInterface getTable(Configuration config, String tableName) throws IOException {
+ return provider.getTable(config, tableName);
+ }
+ public static void addTable(String tableName, String... cf) {
+ provider.addToCache(tableName, cf);
+ }
+ public static MockHTable getTable(String tableName) {
+ try {
+ return (MockHTable) provider.getTable(null, tableName);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to get table: " + tableName);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index b42c2ed..9efcd56 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -32,6 +32,16 @@
<version>${project.parent.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-enrichment</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-pcap</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${global_hadoop_version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
index addeaab..b68bac5 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
@@ -20,9 +20,17 @@ package org.apache.metron.parsers.bolt;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import org.apache.metron.common.Constants;
import org.apache.metron.common.bolt.ConfiguredParserBolt;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
+import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFacade;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.writer.BulkWriterComponent;
+import org.apache.metron.common.writer.WriterToBulkWriter;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.parsers.filters.Filters;
import org.apache.metron.common.configuration.FieldTransformer;
@@ -34,20 +42,46 @@ import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.common.interfaces.MessageWriter;
import org.json.simple.JSONObject;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
-public class ParserBolt extends ConfiguredParserBolt {
+public class ParserBolt extends ConfiguredParserBolt implements Serializable {
private OutputCollector collector;
private MessageParser<JSONObject> parser;
- private MessageFilter<JSONObject> filter;
- private MessageWriter<JSONObject> writer;
+ private MessageFilter<JSONObject> filter = new GenericMessageFilter();
+ private transient Function<ParserConfigurations, WriterConfiguration> writerTransformer;
+ private BulkMessageWriter<JSONObject> messageWriter;
+ private BulkWriterComponent<JSONObject> writerComponent;
+ private boolean isBulk = false;
+ public ParserBolt( String zookeeperUrl
+ , String sensorType
+ , MessageParser<JSONObject> parser
+ , MessageWriter<JSONObject> writer
+ )
+ {
+ super(zookeeperUrl, sensorType);
+ isBulk = false;
+ this.parser = parser;
+ messageWriter = new WriterToBulkWriter<>(writer);
+ }
- public ParserBolt(String zookeeperUrl, String sensorType, MessageParser<JSONObject> parser, MessageWriter<JSONObject> writer) {
+ public ParserBolt( String zookeeperUrl
+ , String sensorType
+ , MessageParser<JSONObject> parser
+ , BulkMessageWriter<JSONObject> writer
+ )
+ {
super(zookeeperUrl, sensorType);
+ isBulk = true;
this.parser = parser;
- this.writer = writer;
+ messageWriter = writer;
+
+
}
public ParserBolt withMessageFilter(MessageFilter<JSONObject> filter) {
@@ -69,7 +103,24 @@ public class ParserBolt extends ConfiguredParserBolt {
);
}
parser.init();
- writer.init();
+
+ if(isBulk) {
+ writerTransformer = config -> new ParserWriterConfiguration(config);
+ }
+ else {
+ writerTransformer = config -> new SingleBatchConfigurationFacade(new ParserWriterConfiguration(config));
+ }
+ try {
+ messageWriter.init(stormConf, writerTransformer.apply(getConfigurations()));
+ } catch (Exception e) {
+ throw new IllegalStateException("Unable to initialize message writer", e);
+ }
+ this.writerComponent = new BulkWriterComponent<JSONObject>(collector, isBulk, isBulk) {
+ @Override
+ protected Collection<Tuple> createTupleCollection() {
+ return new HashSet<>();
+ }
+ };
SensorParserConfig config = getSensorParserConfig();
if(config != null) {
config.init();
@@ -77,6 +128,7 @@ public class ParserBolt extends ConfiguredParserBolt {
else {
throw new IllegalStateException("Unable to retrieve a parser config for " + getSensorType());
}
+ parser.configure(config.getParserConfig());
}
@@ -86,23 +138,27 @@ public class ParserBolt extends ConfiguredParserBolt {
byte[] originalMessage = tuple.getBinary(0);
SensorParserConfig sensorParserConfig = getSensorParserConfig();
try {
+ boolean ackTuple = true;
if(sensorParserConfig != null) {
List<JSONObject> messages = parser.parse(originalMessage);
for (JSONObject message : messages) {
if (parser.validate(message)) {
if (filter != null && filter.emitTuple(message)) {
+ ackTuple = !isBulk;
message.put(Constants.SENSOR_TYPE, getSensorType());
for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) {
if (handler != null) {
handler.transformAndUpdate(message, sensorParserConfig.getParserConfig());
}
}
- writer.write(getSensorType(), configurations, tuple, message);
+ writerComponent.write(getSensorType(), tuple, message, messageWriter, writerTransformer.apply(getConfigurations()));
}
}
}
}
- collector.ack(tuple);
+ if(ackTuple) {
+ collector.ack(tuple);
+ }
} catch (Throwable ex) {
ErrorUtils.handleError(collector, ex, Constants.ERROR_STREAM);
collector.fail(tuple);
@@ -111,6 +167,6 @@ public class ParserBolt extends ConfiguredParserBolt {
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
-
+ declarer.declareStream(Constants.ERROR_STREAM, new Fields("message"));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java
new file mode 100644
index 0000000..a0097ed
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.csv;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.metron.common.csv.CSVConverter;
+import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.SimpleDateFormat;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class CSVParser extends BasicParser {
+ protected static final Logger LOG = LoggerFactory.getLogger(CSVParser.class);
+ public static final String TIMESTAMP_FORMAT_CONF = "timestampFormat";
+ private transient CSVConverter converter;
+ private SimpleDateFormat timestampFormat;
+ @Override
+ public void configure(Map<String, Object> parserConfig) {
+ converter = new CSVConverter();
+ converter.initialize(parserConfig);
+ Object tsFormatObj = parserConfig.get(TIMESTAMP_FORMAT_CONF);
+ if(tsFormatObj != null) {
+ timestampFormat = new SimpleDateFormat(tsFormatObj.toString());
+ }
+ }
+
+ @Override
+ public void init() {
+
+ }
+
+
+ @Override
+ public List<JSONObject> parse(byte[] rawMessage) {
+ try {
+ String msg = new String(rawMessage, "UTF-8");
+ Map<String, String> value = converter.toMap(msg);
+ if(value != null) {
+ value.put("original_string", msg);
+ Object timestampObj = value.get("timestamp");
+ Long timestamp = null;
+ if(timestampObj == null) {
+ timestamp = System.currentTimeMillis();
+ }
+ else {
+ if(timestampFormat == null) {
+ timestamp = ConversionUtils.convert(timestampObj, Long.class);
+ }
+ else {
+ try {
+ timestamp = timestampFormat.parse(timestampObj.toString()).getTime();
+ }
+ catch(Exception e) {
+ LOG.error("Unable to format " + timestampObj.toString());
+ }
+ }
+ }
+ JSONObject jsonVal = new JSONObject(value);
+ jsonVal.put("timestamp", timestamp);
+ return ImmutableList.of(jsonVal);
+ }
+ else {
+ return Collections.emptyList();
+ }
+ } catch (Exception e) {
+ LOG.error("Unable to parse " + new String(rawMessage), e);
+ return Collections.emptyList();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
index 598de55..ad6068f 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
@@ -20,9 +20,14 @@ package org.apache.metron.parsers.topology;
import backtype.storm.topology.TopologyBuilder;
import org.apache.curator.framework.CuratorFramework;
import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.interfaces.MessageWriter;
import org.apache.metron.common.spout.kafka.SpoutConfig;
import org.apache.metron.common.utils.ReflectionUtils;
+import org.apache.metron.common.writer.AbstractWriter;
import org.apache.metron.parsers.bolt.ParserBolt;
import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.parsers.writer.KafkaWriter;
@@ -45,7 +50,13 @@ public class ParserTopologyBuilder {
) throws Exception {
CuratorFramework client = ConfigurationsUtils.getClient(zookeeperUrl);
client.start();
- SensorParserConfig sensorParserConfig = ConfigurationsUtils.readSensorParserConfigFromZookeeper(sensorType, client);
+ ParserConfigurations configurations = new ParserConfigurations();
+ ConfigurationsUtils.updateParserConfigsFromZookeeper(configurations, client);
+ SensorParserConfig sensorParserConfig = configurations.getSensorParserConfig(sensorType);
+ if(sensorParserConfig == null) {
+ throw new IllegalStateException("Cannot find the parser configuration in zookeeper for " + sensorType + "." +
+ " Please check that it exists in zookeeper by using the zk_load_configs.sh -m DUMP command.");
+ }
client.close();
String sensorTopic = sensorParserConfig.getSensorTopic() != null ? sensorParserConfig.getSensorTopic() : sensorType;
TopologyBuilder builder = new TopologyBuilder();
@@ -56,11 +67,31 @@ public class ParserTopologyBuilder {
.setNumTasks(parserNumTasks);
MessageParser<JSONObject> parser = ReflectionUtils.createInstance(sensorParserConfig.getParserClassName());
parser.configure(sensorParserConfig.getParserConfig());
- KafkaWriter writer = new KafkaWriter(brokerUrl);
- ParserBolt parserBolt = new ParserBolt(zookeeperUrl, sensorType, parser, writer);
+ ParserBolt parserBolt = null;
+ {
+ if(sensorParserConfig.getWriterClassName() == null) {
+ KafkaWriter writer = new KafkaWriter(brokerUrl);
+ writer.configure(sensorType, new ParserWriterConfiguration(configurations));
+ parserBolt = new ParserBolt(zookeeperUrl, sensorType, parser, writer);
+ }
+ else {
+ AbstractWriter writer = ReflectionUtils.createInstance(sensorParserConfig.getWriterClassName());
+ writer.configure(sensorType, new ParserWriterConfiguration(configurations));
+ if(writer instanceof BulkMessageWriter) {
+ parserBolt = new ParserBolt(zookeeperUrl, sensorType, parser, (BulkMessageWriter<JSONObject>)writer);
+ }
+ else if(writer instanceof MessageWriter) {
+ parserBolt = new ParserBolt(zookeeperUrl, sensorType, parser, (MessageWriter<JSONObject>)writer);
+ }
+ else {
+ throw new IllegalStateException("Unable to create parser bolt: writer must be a MessageWriter or a BulkMessageWriter");
+ }
+ }
+ }
builder.setBolt("parserBolt", parserBolt, parserParallelism)
.setNumTasks(spoutNumTasks)
.shuffleGrouping("kafkaSpout");
return builder;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
index d168e29..6262dc1 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
@@ -22,21 +22,50 @@ import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.interfaces.MessageWriter;
+import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.common.writer.AbstractWriter;
import org.json.simple.JSONObject;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
-public class KafkaWriter implements MessageWriter<JSONObject>, Serializable {
-
+public class KafkaWriter extends AbstractWriter implements MessageWriter<JSONObject>, Serializable {
+ public enum Configurations {
+ BROKER("kafka.brokerUrl")
+ ,KEY_SERIALIZER("kafka.keySerializer")
+ ,VALUE_SERIALIZER("kafka.valueSerializer")
+ ,REQUIRED_ACKS("kafka.requiredAcks")
+ ,TOPIC("kafka.topic");
+ ;
+ String key;
+ Configurations(String key) {
+ this.key = key;
+ }
+ public Object get(Map<String, Object> config) {
+ return config.get(key);
+ }
+ public <T> T getAndConvert(Map<String, Object> config, Class<T> clazz) {
+ Object o = get(config);
+ if(o != null) {
+ return ConversionUtils.convert(o, clazz);
+ }
+ return null;
+ }
+ }
private String brokerUrl;
private String keySerializer = "org.apache.kafka.common.serialization.StringSerializer";
private String valueSerializer = "org.apache.kafka.common.serialization.StringSerializer";
private int requiredAcks = 1;
+ private String kafkaTopic = Constants.ENRICHMENT_TOPIC;
private KafkaProducer kafkaProducer;
+ public KafkaWriter() {}
+
+
+
public KafkaWriter(String brokerUrl) {
this.brokerUrl = brokerUrl;
}
@@ -51,11 +80,40 @@ public class KafkaWriter implements MessageWriter<JSONObject>, Serializable {
return this;
}
- public KafkaWriter withRequiredAcks(int requiredAcks) {
+ public KafkaWriter withRequiredAcks(Integer requiredAcks) {
this.requiredAcks = requiredAcks;
return this;
}
+ public KafkaWriter withTopic(String topic) {
+ this.kafkaTopic= topic;
+ return this;
+ }
+ @Override
+ public void configure(String sensorName, WriterConfiguration configuration) {
+ Map<String, Object> configMap = configuration.getSensorConfig(sensorName);
+ String brokerUrl = Configurations.BROKER.getAndConvert(configMap, String.class);
+ if(brokerUrl != null) {
+ this.brokerUrl = brokerUrl;
+ }
+ String keySerializer = Configurations.KEY_SERIALIZER.getAndConvert(configMap, String.class);
+ if(keySerializer != null) {
+ withKeySerializer(keySerializer);
+ }
+ String valueSerializer = Configurations.VALUE_SERIALIZER.getAndConvert(configMap, String.class);
+ if(valueSerializer != null) {
+ withValueSerializer(keySerializer);
+ }
+ Integer requiredAcks = Configurations.REQUIRED_ACKS.getAndConvert(configMap, Integer.class);
+ if(requiredAcks!= null) {
+ withRequiredAcks(requiredAcks);
+ }
+ String topic = Configurations.TOPIC.getAndConvert(configMap, String.class);
+ if(topic != null) {
+ withTopic(topic);
+ }
+ }
+
@Override
public void init() {
Map<String, Object> producerConfig = new HashMap<>();
@@ -68,8 +126,8 @@ public class KafkaWriter implements MessageWriter<JSONObject>, Serializable {
@SuppressWarnings("unchecked")
@Override
- public void write(String sourceType, Configurations configurations, Tuple tuple, JSONObject message) throws Exception {
- kafkaProducer.send(new ProducerRecord<String, String>(Constants.ENRICHMENT_TOPIC, message.toJSONString()));
+ public void write(String sourceType, WriterConfiguration configurations, Tuple tuple, JSONObject message) throws Exception {
+ kafkaProducer.send(new ProducerRecord<String, String>(kafkaTopic, message.toJSONString()));
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
index 658d52e..73aad23 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
@@ -17,6 +17,16 @@
*/
package org.apache.metron.parsers.bolt;
+import org.apache.metron.common.configuration.SensorParserConfig;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.tuple.Tuple;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.metron.common.configuration.ParserConfigurations;
@@ -28,6 +38,7 @@ import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.common.interfaces.MessageWriter;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
+import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mock;
@@ -35,6 +46,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
@@ -52,18 +64,42 @@ public class ParserBoltTest extends BaseBoltTest {
private MessageWriter<JSONObject> writer;
@Mock
+ private BulkMessageWriter<JSONObject> batchWriter;
+
+ @Mock
private MessageFilter<JSONObject> filter;
+ @Mock
+ private Tuple t1;
+
+ @Mock
+ private Tuple t2;
+
+ @Mock
+ private Tuple t3;
+
+ @Mock
+ private Tuple t4;
+
+ @Mock
+ private Tuple t5;
+
@Test
public void test() throws Exception {
String sensorType = "yaf";
ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, writer) {
@Override
- public ParserConfigurations getConfigurations() {
+ protected ParserConfigurations defaultConfigurations() {
return new ParserConfigurations() {
@Override
public SensorParserConfig getSensorParserConfig(String sensorType) {
- return new SensorParserConfig();
+ return new SensorParserConfig() {
+ @Override
+ public Map<String, Object> getParserConfig() {
+ return new HashMap<String, Object>() {{
+ }};
+ }
+ };
}
};
}
@@ -88,7 +124,7 @@ public class ParserBoltTest extends BaseBoltTest {
when(parser.validate(eq(messages.get(0)))).thenReturn(true);
when(parser.validate(eq(messages.get(1)))).thenReturn(false);
parserBolt.execute(tuple);
- verify(writer, times(1)).write(eq(sensorType), any(Configurations.class), eq(tuple), eq(finalMessage1));
+ verify(writer, times(1)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage1));
verify(outputCollector, times(1)).ack(tuple);
when(parser.validate(eq(messages.get(0)))).thenReturn(true);
when(parser.validate(eq(messages.get(1)))).thenReturn(true);
@@ -96,12 +132,46 @@ public class ParserBoltTest extends BaseBoltTest {
when(filter.emitTuple(messages.get(1))).thenReturn(true);
parserBolt.withMessageFilter(filter);
parserBolt.execute(tuple);
- verify(writer, times(1)).write(eq(sensorType), any(Configurations.class), eq(tuple), eq(finalMessage2));
+ verify(writer, times(1)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage2));
verify(outputCollector, times(2)).ack(tuple);
- doThrow(new Exception()).when(writer).write(eq(sensorType), any(Configurations.class), eq(tuple), eq(finalMessage2));
+ doThrow(new Exception()).when(writer).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage2));
parserBolt.execute(tuple);
verify(outputCollector, times(1)).reportError(any(Throwable.class));
}
+@Test
+public void testImplicitBatchOfOne() throws Exception {
+
+ String sensorType = "yaf";
+
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, batchWriter) {
+ @Override
+ protected ParserConfigurations defaultConfigurations() {
+ return new ParserConfigurations() {
+ @Override
+ public SensorParserConfig getSensorParserConfig(String sensorType) {
+ return new SensorParserConfig() {
+ @Override
+ public Map<String, Object> getParserConfig() {
+ return new HashMap<String, Object>() {{
+ }};
+ }
+ };
+ }
+ };
+ }
+ };
+ parserBolt.setCuratorFramework(client);
+ parserBolt.setTreeCache(cache);
+ parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
+ verify(parser, times(1)).init();
+ verify(batchWriter, times(1)).init(any(), any());
+ when(parser.validate(any())).thenReturn(true);
+ when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject()));
+ when(filter.emitTuple(any())).thenReturn(true);
+ parserBolt.withMessageFilter(filter);
+ parserBolt.execute(t1);
+ verify(outputCollector, times(1)).ack(t1);
+}
/**
{
@@ -116,7 +186,7 @@ public class ParserBoltTest extends BaseBoltTest {
@Test
public void testFilter() throws Exception {
String sensorType = "yaf";
- ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, writer) {
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, batchWriter) {
@Override
protected SensorParserConfig getSensorParserConfig() {
try {
@@ -130,6 +200,150 @@ public class ParserBoltTest extends BaseBoltTest {
parserBolt.setTreeCache(cache);
parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
verify(parser, times(1)).init();
+ verify(batchWriter, times(1)).init(any(), any());
+ when(parser.validate(any())).thenReturn(true);
+ when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject()));
+ parserBolt.withMessageFilter(filter);
+ parserBolt.execute(t1);
+ verify(outputCollector, times(1)).ack(t1);
+ }
+ @Test
+ public void testBatchOfOne() throws Exception {
+
+ String sensorType = "yaf";
+
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, batchWriter) {
+ @Override
+ protected ParserConfigurations defaultConfigurations() {
+ return new ParserConfigurations() {
+ @Override
+ public SensorParserConfig getSensorParserConfig(String sensorType) {
+ return new SensorParserConfig() {
+ @Override
+ public Map<String, Object> getParserConfig() {
+ return new HashMap<String, Object>() {{
+ put(ParserWriterConfiguration.BATCH_CONF, "1");
+ }};
+ }
+ };
+ }
+ };
+ }
+ };
+ parserBolt.setCuratorFramework(client);
+ parserBolt.setTreeCache(cache);
+ parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
+ verify(parser, times(1)).init();
+ verify(batchWriter, times(1)).init(any(), any());
+ when(parser.validate(any())).thenReturn(true);
+ when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject()));
+ when(filter.emitTuple(any())).thenReturn(true);
+ parserBolt.withMessageFilter(filter);
+ parserBolt.execute(t1);
+ verify(outputCollector, times(1)).ack(t1);
+ }
+ @Test
+ public void testBatchOfFive() throws Exception {
+
+ String sensorType = "yaf";
+
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, batchWriter) {
+ @Override
+ protected ParserConfigurations defaultConfigurations() {
+ return new ParserConfigurations() {
+ @Override
+ public SensorParserConfig getSensorParserConfig(String sensorType) {
+ return new SensorParserConfig() {
+ @Override
+ public Map<String, Object> getParserConfig() {
+ return new HashMap<String, Object>() {{
+ put(ParserWriterConfiguration.BATCH_CONF, 5);
+ }};
+ }
+ };
+ }
+ };
+ }
+ };
+ parserBolt.setCuratorFramework(client);
+ parserBolt.setTreeCache(cache);
+ parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
+ verify(parser, times(1)).init();
+ verify(batchWriter, times(1)).init(any(), any());
+ when(parser.validate(any())).thenReturn(true);
+ when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject()));
+ when(filter.emitTuple(any())).thenReturn(true);
+ parserBolt.withMessageFilter(filter);
+ writeNonBatch(outputCollector, parserBolt, t1);
+ writeNonBatch(outputCollector, parserBolt, t2);
+ writeNonBatch(outputCollector, parserBolt, t3);
+ writeNonBatch(outputCollector, parserBolt, t4);
+ parserBolt.execute(t5);
+ verify(outputCollector, times(1)).ack(t1);
+ verify(outputCollector, times(1)).ack(t2);
+ verify(outputCollector, times(1)).ack(t3);
+ verify(outputCollector, times(1)).ack(t4);
+ verify(outputCollector, times(1)).ack(t5);
+
+
+ }
+ @Test
+ public void testBatchOfFiveWithError() throws Exception {
+
+ String sensorType = "yaf";
+ ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, batchWriter) {
+ @Override
+ protected ParserConfigurations defaultConfigurations() {
+ return new ParserConfigurations() {
+ @Override
+ public SensorParserConfig getSensorParserConfig(String sensorType) {
+ return new SensorParserConfig() {
+ @Override
+ public Map<String, Object> getParserConfig() {
+ return new HashMap<String, Object>() {{
+ put(ParserWriterConfiguration.BATCH_CONF, 5);
+ }};
+ }
+ };
+ }
+ };
+ }
+ };
+ parserBolt.setCuratorFramework(client);
+ parserBolt.setTreeCache(cache);
+ parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
+ verify(parser, times(1)).init();
+ verify(batchWriter, times(1)).init(any(), any());
+
+ doThrow(new Exception()).when(batchWriter).write(any(), any(), any(), any());
+ when(parser.validate(any())).thenReturn(true);
+ when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject()));
+ when(filter.emitTuple(any())).thenReturn(true);
+ parserBolt.withMessageFilter(filter);
+ writeNonBatch(outputCollector, parserBolt, t1);
+ writeNonBatch(outputCollector, parserBolt, t2);
+ writeNonBatch(outputCollector, parserBolt, t3);
+ writeNonBatch(outputCollector, parserBolt, t4);
+ parserBolt.execute(t5);
+ verify(outputCollector, times(0)).ack(t1);
+ verify(outputCollector, times(1)).fail(t1);
+ verify(outputCollector, times(0)).ack(t2);
+ verify(outputCollector, times(1)).fail(t2);
+ verify(outputCollector, times(0)).ack(t3);
+ verify(outputCollector, times(1)).fail(t3);
+ verify(outputCollector, times(0)).ack(t4);
+ verify(outputCollector, times(1)).fail(t4);
+ verify(outputCollector, times(0)).ack(t5);
+ verify(outputCollector, times(1)).fail(t5);
+
+
+ }
+ private static void writeNonBatch(OutputCollector collector, ParserBolt bolt, Tuple t) {
+ bolt.execute(t);
+ verify(collector, times(0)).ack(t);
+ }
+
+/*=======
verify(writer, times(1)).init();
byte[] sampleBinary = "some binary message".getBytes();
JSONParser jsonParser = new JSONParser();
@@ -147,4 +361,5 @@ public class ParserBoltTest extends BaseBoltTest {
verify(writer, times(1)).write(eq(sensorType), any(Configurations.class), eq(tuple), eq(finalMessage1));
verify(outputCollector, times(1)).ack(tuple);
}
+>>>>>>> master*/
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java
new file mode 100644
index 0000000..5f314fa
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.csv;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.utils.JSONUtils;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+public class CSVParserTest {
+ /**
+ {
+ "parserClassName" : "org.apache.metron.parsers.csv.CSVParser"
+ ,"sensorTopic":"dummy"
+ ,"parserConfig":
+ {
+ "columns" : {
+ "col1" : 0
+ ,"col2" : 1
+ ,"col3" : 2
+ }
+ }
+ }
+ */
+ @Multiline
+ public static String parserConfig;
+
+ @Test
+ public void test() throws IOException {
+ CSVParser parser = new CSVParser();
+
+ SensorParserConfig config = JSONUtils.INSTANCE.load(parserConfig, SensorParserConfig.class);
+ parser.init();
+ parser.configure(config.getParserConfig());
+ {
+ String line = "#foo,bar,grok";
+ Assert.assertEquals(0, parser.parse(Bytes.toBytes(line)).size());
+ }
+ {
+ String line = "";
+ Assert.assertEquals(0, parser.parse(Bytes.toBytes(line)).size());
+ }
+ {
+ String line = "foo,bar,grok";
+ List<JSONObject> results = parser.parse(Bytes.toBytes(line));
+ Assert.assertEquals(1, results.size());
+ JSONObject o = results.get(0);
+ Assert.assertTrue(parser.validate(o));
+ Assert.assertEquals(5, o.size());
+ Assert.assertEquals("foo", o.get("col1"));
+ Assert.assertEquals("bar", o.get("col2"));
+ Assert.assertEquals("grok", o.get("col3"));
+ }
+ {
+ String line = "\"foo\", \"bar\",\"grok\"";
+ List<JSONObject> results = parser.parse(Bytes.toBytes(line));
+ Assert.assertEquals(1, results.size());
+ JSONObject o = results.get(0);
+ Assert.assertTrue(parser.validate(o));
+ Assert.assertEquals(5, o.size());
+ Assert.assertEquals("foo", o.get("col1"));
+ Assert.assertEquals("bar", o.get("col2"));
+ Assert.assertEquals("grok", o.get("col3"));
+ }
+ {
+ String line = "foo, bar, grok";
+ List<JSONObject> results = parser.parse(Bytes.toBytes(line));
+ Assert.assertEquals(1, results.size());
+ JSONObject o = results.get(0);
+ Assert.assertTrue(parser.validate(o));
+ Assert.assertEquals(5, o.size());
+ Assert.assertEquals("foo", o.get("col1"));
+ Assert.assertEquals(" bar", o.get("col2"));
+ Assert.assertEquals(" grok", o.get("col3"));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
new file mode 100644
index 0000000..cea635f
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writers;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.enrichment.converter.EnrichmentConverter;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.lookup.LookupKV;
+import org.apache.metron.integration.mock.MockTableProvider;
+import org.apache.metron.test.mock.MockHTable;
+import org.apache.metron.writer.hbase.SimpleHbaseEnrichmentWriter;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class SimpleHBaseEnrichmentWriterTest {
+ private static final String SENSOR_TYPE= "dummy";
+ private static final String TABLE_NAME= SENSOR_TYPE;
+ private static final String TABLE_CF= "cf";
+ private static final String ENRICHMENT_TYPE = "et";
+ private static final Map<String, Object> BASE_WRITER_CONFIG = new HashMap<String, Object>() {{
+ put(SimpleHbaseEnrichmentWriter.Configurations.HBASE_TABLE.getKey(), TABLE_NAME);
+ put(SimpleHbaseEnrichmentWriter.Configurations.HBASE_CF.getKey(), TABLE_CF);
+ put(SimpleHbaseEnrichmentWriter.Configurations.ENRICHMENT_TYPE.getKey(), ENRICHMENT_TYPE);
+ put(SimpleHbaseEnrichmentWriter.Configurations.HBASE_PROVIDER.getKey(), MockTableProvider.class.getName());
+ }};
+ @Before
+ public void setupMockTable() {
+ MockTableProvider.addTable(TABLE_NAME, TABLE_CF);
+ }
+ @Test
+ public void testBatchOneNormalPath() throws Exception {
+ final String sensorType = "dummy";
+ SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
+
+ WriterConfiguration configuration = createConfig(1,
+ new HashMap<String, Object>(BASE_WRITER_CONFIG) {{
+ put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
+ }}
+ );
+ writer.configure(sensorType,configuration);
+
+ writer.write( SENSOR_TYPE
+ , configuration
+ , null
+ , new ArrayList<JSONObject>() {{
+ add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar")));
+ }}
+ );
+ List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues();
+ Assert.assertEquals(1, values.size());
+ Assert.assertEquals("localhost", values.get(0).getKey().indicator);
+ Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user"));
+ Assert.assertEquals("bar", values.get(0).getValue().getMetadata().get("foo"));
+ Assert.assertEquals(2, values.get(0).getValue().getMetadata().size());
+ }
+
+ @Test
+ public void testFilteredKey() throws Exception {
+ final String sensorType = "dummy";
+ SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
+
+ WriterConfiguration configuration = createConfig(1,
+ new HashMap<String, Object>(BASE_WRITER_CONFIG) {{
+ put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
+ put(SimpleHbaseEnrichmentWriter.Configurations.VALUE_COLUMNS.getKey(), "user");
+ }}
+ );
+ writer.configure(sensorType,configuration);
+
+ writer.write( SENSOR_TYPE
+ , configuration
+ , null
+ , new ArrayList<JSONObject>() {{
+ add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar")));
+ }}
+ );
+ List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues();
+ Assert.assertEquals(1, values.size());
+ Assert.assertEquals("localhost", values.get(0).getKey().indicator);
+ Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user"));
+ Assert.assertNull(values.get(0).getValue().getMetadata().get("foo"));
+ Assert.assertEquals(1, values.get(0).getValue().getMetadata().size());
+ }
+
+ @Test
+ public void testFilteredKeys() throws Exception {
+ final String sensorType = "dummy";
+ SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
+
+ WriterConfiguration configuration = createConfig(1,
+ new HashMap<String, Object>(BASE_WRITER_CONFIG) {{
+ put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
+ put(SimpleHbaseEnrichmentWriter.Configurations.VALUE_COLUMNS.getKey(), ImmutableList.of("user", "ip"));
+ }}
+ );
+ writer.configure(sensorType,configuration);
+
+ writer.write( SENSOR_TYPE
+ , configuration
+ , null
+ , new ArrayList<JSONObject>() {{
+ add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar")));
+ }}
+ );
+ List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues();
+ Assert.assertEquals(1, values.size());
+ Assert.assertEquals("localhost", values.get(0).getKey().indicator);
+ Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user"));
+ Assert.assertEquals("localhost", values.get(0).getValue().getMetadata().get("ip"));
+ Assert.assertNull(values.get(0).getValue().getMetadata().get("foo"));
+ Assert.assertEquals(2, values.get(0).getValue().getMetadata().size());
+ }
+ public static List<LookupKV<EnrichmentKey, EnrichmentValue>> getValues() throws IOException {
+ MockHTable table = MockTableProvider.getTable(TABLE_NAME);
+ Assert.assertNotNull(table);
+ List<LookupKV<EnrichmentKey, EnrichmentValue>> ret = new ArrayList<>();
+ EnrichmentConverter converter = new EnrichmentConverter();
+ for(Result r : table.getScanner(Bytes.toBytes(TABLE_CF))) {
+ ret.add(converter.fromResult(r, TABLE_CF));
+ }
+ return ret;
+ }
+ public static WriterConfiguration createConfig(final int batchSize, final Map<String, Object> sensorConfig)
+ {
+ return new WriterConfiguration() {
+ @Override
+ public int getBatchSize(String sensorName) {
+ return batchSize;
+ }
+
+ @Override
+ public String getIndex(String sensorName) {
+ return SENSOR_TYPE;
+ }
+
+ @Override
+ public Map<String, Object> getSensorConfig(String sensorName) {
+ return sensorConfig;
+
+ }
+
+ @Override
+ public Map<String, Object> getGlobalConfig() {
+ return null;
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
new file mode 100644
index 0000000..f6436b9
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writers.integration;
+
+import com.google.common.collect.ImmutableList;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.TestConstants;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.enrichment.converter.EnrichmentConverter;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.lookup.LookupKV;
+import org.apache.metron.integration.*;
+import org.apache.metron.integration.components.ConfigUploadComponent;
+import org.apache.metron.integration.components.KafkaWithZKComponent;
+import org.apache.metron.integration.mock.MockTableProvider;
+import org.apache.metron.integration.utils.TestUtils;
+import org.apache.metron.parsers.integration.components.ParserTopologyComponent;
+import org.apache.metron.test.TestDataType;
+import org.apache.metron.test.mock.MockHTable;
+import org.apache.metron.test.utils.SampleDataUtils;
+import org.apache.metron.test.utils.UnitTestHelper;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.*;
+
+public class SimpleHbaseEnrichmentWriterIntegrationTest extends BaseIntegrationTest {
+
+ /**
+ {
+ "parserClassName" : "org.apache.metron.parsers.csv.CSVParser"
+ ,"writerClassName" : "org.apache.metron.writer.hbase.SimpleHbaseEnrichmentWriter"
+ ,"sensorTopic":"dummy"
+ ,"parserConfig":
+ {
+ "shew.table" : "dummy"
+ ,"shew.cf" : "cf"
+ ,"shew.keyColumns" : "col2"
+ ,"shew.enrichmentType" : "et"
+ ,"shew.hbaseProvider" : "org.apache.metron.integration.mock.MockTableProvider"
+ ,"columns" : {
+ "col1" : 0
+ ,"col2" : 1
+ ,"col3" : 2
+ }
+ }
+ }
+ */
+ @Multiline
+ public static String parserConfig;
+
+ @Test
+ public void test() throws UnableToStartException, IOException {
+ final String sensorType = "dummy";
+ final List<byte[]> inputMessages = new ArrayList<byte[]>() {{
+ add(Bytes.toBytes("col11,col12,col13"));
+ add(Bytes.toBytes("col21,col22,col23"));
+ add(Bytes.toBytes("col31,col32,col33"));
+ }};
+ MockTableProvider.addTable(sensorType, "cf");
+ final Properties topologyProperties = new Properties();
+ final KafkaWithZKComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaWithZKComponent.Topic>() {{
+ add(new KafkaWithZKComponent.Topic(sensorType, 1));
+ }});
+ topologyProperties.setProperty("kafka.broker", kafkaComponent.getBrokerList());
+
+ ConfigUploadComponent configUploadComponent = new ConfigUploadComponent()
+ .withTopologyProperties(topologyProperties)
+ .withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH)
+ .withParserSensorConfig(sensorType, JSONUtils.INSTANCE.load(parserConfig, SensorParserConfig.class));
+
+ ParserTopologyComponent parserTopologyComponent = new ParserTopologyComponent.Builder()
+ .withSensorType(sensorType)
+ .withTopologyProperties(topologyProperties)
+ .withBrokerUrl(kafkaComponent.getBrokerList()).build();
+
+ UnitTestHelper.verboseLogging();
+ ComponentRunner runner = new ComponentRunner.Builder()
+ .withComponent("kafka", kafkaComponent)
+ .withComponent("config", configUploadComponent)
+ .withComponent("storm", parserTopologyComponent)
+ .withMillisecondsBetweenAttempts(5000)
+ .withNumRetries(10)
+ .build();
+ try {
+ runner.start();
+ kafkaComponent.writeMessages(sensorType, inputMessages);
+ List<LookupKV<EnrichmentKey, EnrichmentValue>> outputMessages =
+ runner.process(new Processor<List<LookupKV<EnrichmentKey, EnrichmentValue>>>() {
+ List<LookupKV<EnrichmentKey, EnrichmentValue>> messages = null;
+
+ public ReadinessState process(ComponentRunner runner) {
+ MockHTable table = MockTableProvider.getTable(sensorType);
+ if (table != null && table.size() == inputMessages.size()) {
+ EnrichmentConverter converter = new EnrichmentConverter();
+ messages = new ArrayList<>();
+ try {
+ for (Result r : table.getScanner(Bytes.toBytes("cf"))) {
+ messages.add(converter.fromResult(r, "cf"));
+ }
+ } catch (IOException e) {
+ }
+ return ReadinessState.READY;
+ }
+ return ReadinessState.NOT_READY;
+ }
+
+ public List<LookupKV<EnrichmentKey, EnrichmentValue>> getResult() {
+ return messages;
+ }
+ });
+ Set<String> validIndicators = new HashSet<>(ImmutableList.of("col12", "col22", "col32"));
+ Map<String, Map<String, String>> validMetadata = new HashMap<String, Map<String, String>>() {{
+ put("col12", new HashMap<String, String>() {{
+ put("col1", "col11");
+ put("col3", "col13");
+ }});
+ put("col22", new HashMap<String, String>() {{
+ put("col1", "col21");
+ put("col3", "col23");
+ }});
+ put("col32", new HashMap<String, String>() {{
+ put("col1", "col31");
+ put("col3", "col33");
+ }});
+ }};
+ for (LookupKV<EnrichmentKey, EnrichmentValue> kv : outputMessages) {
+ Assert.assertTrue(validIndicators.contains(kv.getKey().indicator));
+ Assert.assertEquals(kv.getValue().getMetadata().get("source.type"), "dummy");
+ Assert.assertNotNull(kv.getValue().getMetadata().get("timestamp"));
+ Assert.assertNotNull(kv.getValue().getMetadata().get("original_string"));
+ Map<String, String> metadata = validMetadata.get(kv.getKey().indicator);
+ for (Map.Entry<String, String> x : metadata.entrySet()) {
+ Assert.assertEquals(kv.getValue().getMetadata().get(x.getKey()), x.getValue());
+ }
+ Assert.assertEquals(metadata.size() + 3, kv.getValue().getMetadata().size());
+ }
+ }
+ finally {
+ if(runner != null) {
+ runner.stop();
+ }
+ }
+ }
+}