You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/05/31 21:00:28 UTC

[2/3] incubator-metron git commit: METRON-174 Storm consumption of hbase enrichment reference data. This closes apache/incubator-metron#127

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java
index d9b7b38..bd019a0 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/converter/EnrichmentValue.java
@@ -35,20 +35,20 @@ public class EnrichmentValue implements LookupValue {
     public static final String VALUE_COLUMN_NAME = "v";
     public static final byte[] VALUE_COLUMN_NAME_B = Bytes.toBytes(VALUE_COLUMN_NAME);
 
-    private Map<String, String> metadata = null;
+    private Map<String, Object> metadata = null;
 
     public EnrichmentValue()
     {
 
     }
 
-    public EnrichmentValue(Map<String, String> metadata) {
+    public EnrichmentValue(Map<String, Object> metadata) {
         this.metadata = metadata;
     }
 
 
 
-    public Map<String, String> getMetadata() {
+    public Map<String, Object> getMetadata() {
         return metadata;
     }
 
@@ -66,14 +66,14 @@ public class EnrichmentValue implements LookupValue {
             }
         }
     }
-    public Map<String, String> stringToValue(String s){
+    public Map<String, Object> stringToValue(String s){
         try {
-            return _mapper.get().readValue(s, new TypeReference<Map<String, String>>(){});
+            return _mapper.get().readValue(s, new TypeReference<Map<String, Object>>(){});
         } catch (IOException e) {
             throw new RuntimeException("Unable to convert string to metadata: " + s);
         }
     }
-    public String valueToString(Map<String, String> value) {
+    public String valueToString(Map<String, Object> value) {
         try {
             return _mapper.get().writeValueAsString(value);
         } catch (IOException e) {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
new file mode 100644
index 0000000..f7e7aeb
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
@@ -0,0 +1,282 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.hbase;
+
+import backtype.storm.tuple.Tuple;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.common.utils.ReflectionUtils;
+import org.apache.metron.common.writer.AbstractWriter;
+import org.apache.metron.enrichment.converter.EnrichmentConverter;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.converter.HbaseConverter;
+import org.apache.metron.hbase.HTableProvider;
+import org.apache.metron.hbase.TableProvider;
+import org.json.simple.JSONObject;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public class SimpleHbaseEnrichmentWriter extends AbstractWriter implements BulkMessageWriter<JSONObject>, Serializable {
+  public enum Configurations {
+    HBASE_TABLE("shew.table")
+    ,HBASE_CF("shew.cf")
+    ,KEY_COLUMNS("shew.keyColumns")
+    ,KEY_DELIM("shew.keyDelim")
+    ,ENRICHMENT_TYPE("shew.enrichmentType")
+    ,VALUE_COLUMNS("shew.valueColumns")
+    ,HBASE_PROVIDER("shew.hbaseProvider")
+    ;
+    String key;
+    Configurations(String key) {
+      this.key = key;
+    }
+    public String getKey() {
+      return key;
+    }
+    public Object get(Map<String, Object> config) {
+      return config.get(key);
+    }
+    public <T> T getAndConvert(Map<String, Object> config, Class<T> clazz) {
+      Object o = get(config);
+      if(o != null) {
+        return ConversionUtils.convert(o, clazz);
+      }
+      return null;
+    }
+  }
+  public static class KeyTransformer {
+    List<String> keys = new ArrayList<>();
+    Set<String> keySet;
+    private String delim = ":";
+    public KeyTransformer(String key) {
+      this(key, null);
+    }
+    public KeyTransformer(String key, String delim) {
+      keys.add(key);
+      keySet = new HashSet<>(this.keys);
+      this.delim = delim == null?this.delim:delim;
+    }
+    public KeyTransformer(Iterable<String> keys) {
+      this(keys, null);
+    }
+    public KeyTransformer(Iterable<String> keys, String delim) {
+      Iterables.addAll(this.keys, keys);
+      keySet = new HashSet<>(this.keys);
+      this.delim = delim == null?this.delim:delim;
+    }
+
+    public String transform(final JSONObject message) {
+      return
+      keys.stream().map( x -> {
+        Object o = message.get(x);
+        return o == null?"":o.toString();
+      }).collect(Collectors.joining(delim));
+    }
+  }
+  private transient EnrichmentConverter converter;
+  private String tableName;
+  private String cf;
+  private HTableInterface table;
+  private TableProvider provider;
+  private Map.Entry<Object, KeyTransformer> keyTransformer;
+
+  public SimpleHbaseEnrichmentWriter() {
+  }
+
+  @Override
+  public void configure(String sensorName, WriterConfiguration configuration) {
+    String hbaseProviderImpl = Configurations.HBASE_PROVIDER.getAndConvert(configuration.getSensorConfig(sensorName),String.class);
+    if(hbaseProviderImpl != null) {
+      provider = ReflectionUtils.createInstance(hbaseProviderImpl);
+    }
+    if(converter == null) {
+      converter = new EnrichmentConverter();
+    }
+  }
+
+  @Override
+  public void init(Map stormConf, WriterConfiguration configuration) throws Exception {
+    if(converter == null) {
+      converter = new EnrichmentConverter();
+    }
+  }
+
+  protected synchronized TableProvider getProvider() {
+    if(provider == null) {
+
+      provider = new HTableProvider();
+    }
+    return provider;
+  }
+
+  public HTableInterface getTable(String tableName, String cf) throws IOException {
+    synchronized(this) {
+      boolean isInitial = this.tableName == null || this.cf == null;
+      boolean isValid = tableName != null && cf != null;
+
+      if( isInitial || (isValid && (!this.tableName.equals(tableName) || !this.cf.equals(cf)) )
+        )
+      {
+        Configuration conf = HBaseConfiguration.create();
+        //new table connection
+        if(table != null) {
+          table.close();
+        }
+        table = getProvider().getTable(conf, tableName);
+        this.tableName = tableName;
+        this.cf = cf;
+      }
+      return table;
+    }
+  }
+
+  public HTableInterface getTable(Map<String, Object> config) throws IOException {
+    return getTable(Configurations.HBASE_TABLE.getAndConvert(config, String.class)
+                   ,Configurations.HBASE_CF.getAndConvert(config, String.class)
+                   );
+
+  }
+
+
+  private List<String> getColumns(Object keyColumnsObj, boolean allowNull) {
+    Object o = keyColumnsObj;
+    if(allowNull && keyColumnsObj == null) {
+      return Collections.emptyList();
+    }
+    if(o instanceof String) {
+      return ImmutableList.of(o.toString());
+    }
+    else if (o instanceof List) {
+      List<String> keyCols = new ArrayList<>();
+      for(Object key : (List)o) {
+        keyCols.add(key.toString());
+      }
+      return keyCols;
+    }
+    else {
+      throw new RuntimeException("Unable to get columns: " + o);
+    }
+  }
+
+  private KeyTransformer getTransformer(Map<String, Object> config) {
+    Object o = Configurations.KEY_COLUMNS.get(config);
+    KeyTransformer transformer = null;
+    if(keyTransformer != null && keyTransformer.getKey() == o) {
+      return keyTransformer.getValue();
+    }
+    else {
+      List<String> keys = getColumns(o, false);
+      Object delimObj = Configurations.KEY_DELIM.get(config);
+      String delim = (delimObj == null || !(delimObj instanceof String))?null:delimObj.toString();
+      transformer = new KeyTransformer(keys, delim);
+      keyTransformer = new AbstractMap.SimpleEntry<>(o, transformer);
+      return transformer;
+    }
+  }
+
+
+  private EnrichmentValue getValue( JSONObject message
+                                  , Set<String> keyColumns
+                                  , Set<String> valueColumns
+                                  )
+  {
+    Map<String, Object> metadata = new HashMap<>();
+    if(valueColumns == null || valueColumns.isEmpty()) {
+      for (Object kv : message.entrySet()) {
+        Map.Entry<Object, Object> entry = (Map.Entry<Object, Object>) kv;
+        if (!keyColumns.contains(entry.getKey())) {
+          metadata.put(entry.getKey().toString(), entry.getValue());
+        }
+      }
+      return new EnrichmentValue(metadata);
+    }
+    else {
+      for (Object kv : message.entrySet()) {
+        Map.Entry<Object, Object> entry = (Map.Entry<Object, Object>) kv;
+        if (valueColumns.contains(entry.getKey())) {
+          metadata.put(entry.getKey().toString(), entry.getValue());
+        }
+      }
+      return new EnrichmentValue(metadata);
+    }
+  }
+
+  private EnrichmentKey getKey(JSONObject message, KeyTransformer transformer, String enrichmentType) {
+    if(enrichmentType != null) {
+      return new EnrichmentKey(enrichmentType, transformer.transform(message));
+    }
+    else {
+      return null;
+    }
+  }
+
+  @Override
+  public void write( String sensorType
+                    , WriterConfiguration configurations
+                    , Iterable<Tuple> tuples
+                    , List<JSONObject> messages
+                    ) throws Exception
+  {
+    Map<String, Object> sensorConfig = configurations.getSensorConfig(sensorType);
+    HTableInterface table = getTable(sensorConfig);
+    KeyTransformer transformer = getTransformer(sensorConfig);
+    Object enrichmentTypeObj = Configurations.ENRICHMENT_TYPE.get(sensorConfig);
+    String enrichmentType = enrichmentTypeObj == null?null:enrichmentTypeObj.toString();
+    Set<String> valueColumns = new HashSet<>(getColumns(Configurations.VALUE_COLUMNS.get(sensorConfig), true));
+    List<Put> puts = new ArrayList<>();
+    for(JSONObject message : messages) {
+      EnrichmentKey key = getKey(message, transformer, enrichmentType);
+      EnrichmentValue value = getValue(message, transformer.keySet, valueColumns);
+      if(key == null || value == null) {
+        continue;
+      }
+      Put put = converter.toPut(this.cf, key, value);
+      if(put != null) {
+        puts.add(put);
+      }
+    }
+    table.put(puts);
+  }
+
+  @Override
+  public void close() throws Exception {
+    synchronized(this) {
+      if(table != null) {
+        table.close();
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
index d8f0e73..01f1245 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
@@ -19,6 +19,7 @@ package org.apache.metron.writer.hdfs;
 
 import backtype.storm.tuple.Tuple;
 import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
 import org.apache.metron.common.interfaces.BulkMessageWriter;
 import org.apache.storm.hdfs.bolt.format.FileNameFormat;
 import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
@@ -62,14 +63,15 @@ public class HdfsWriter implements BulkMessageWriter<JSONObject>, Serializable {
   }
 
   @Override
-  public void init(Map stormConfig, EnrichmentConfigurations configurations) {
+  public void init(Map stormConfig, WriterConfiguration configurations) {
     this.stormConfig = stormConfig;
   }
 
+
   @Override
   public void write(String sourceType
-                   , EnrichmentConfigurations configurations
-                   , List<Tuple> tuples
+                   , WriterConfiguration configurations
+                   , Iterable<Tuple> tuples
                    , List<JSONObject> messages
                    ) throws Exception
   {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
index 1a958df..bf72219 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapterTest.java
@@ -50,10 +50,10 @@ public class SimpleHBaseAdapterTest {
   private EnrichmentLookup lookup;
   private static final String PLAYFUL_CLASSIFICATION_TYPE = "playful_classification";
   private static final String CF1_CLASSIFICATION_TYPE = "cf1";
-  private static final Map<String, String> CF1_ENRICHMENT = new HashMap<String, String>() {{
+  private static final Map<String, Object> CF1_ENRICHMENT = new HashMap<String, Object>() {{
     put("key", "value");
   }};
-  private static final Map<String, String> PLAYFUL_ENRICHMENT = new HashMap<String, String>() {{
+  private static final Map<String, Object> PLAYFUL_ENRICHMENT = new HashMap<String, Object>() {{
     put("orientation", "north");
   }};
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
index f420b01..46216b2 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapterTest.java
@@ -101,7 +101,7 @@ public class ThreatIntelAdapterTest {
     final MockHTable trackerTable = (MockHTable) MockHTable.Provider.addToCache(atTableName, cf);
     final MockHTable threatIntelTable = (MockHTable) MockHTable.Provider.addToCache(threatIntelTableName, cf);
     EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {{
-      add(new LookupKV<>(new EnrichmentKey("10.0.2.3", "10.0.2.3"), new EnrichmentValue(new HashMap<String, String>())));
+      add(new LookupKV<>(new EnrichmentKey("10.0.2.3", "10.0.2.3"), new EnrichmentValue(new HashMap<>())));
     }});
 
     BloomAccessTracker bat = new BloomAccessTracker(threatIntelTableName, 100, 0.03);

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
index 61961a7..332ded0 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
@@ -22,6 +22,7 @@ import backtype.storm.tuple.Values;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
 import org.apache.metron.test.bolt.BaseEnrichmentBoltTest;
 import org.apache.metron.common.configuration.Configurations;
 import org.apache.metron.common.interfaces.BulkMessageWriter;
@@ -117,28 +118,28 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
     bulkMessageWriterBolt.declareOutputFields(declarer);
     verify(declarer, times(1)).declareStream(eq("error"), argThat(new FieldsMatcher("message")));
     Map stormConf = new HashMap();
-    doThrow(new Exception()).when(bulkMessageWriter).init(eq(stormConf), any(EnrichmentConfigurations.class));
+    doThrow(new Exception()).when(bulkMessageWriter).init(eq(stormConf), any(WriterConfiguration.class));
     try {
       bulkMessageWriterBolt.prepare(stormConf, topologyContext, outputCollector);
       fail("A runtime exception should be thrown when bulkMessageWriter.init throws an exception");
     } catch(RuntimeException e) {}
     reset(bulkMessageWriter);
     bulkMessageWriterBolt.prepare(stormConf, topologyContext, outputCollector);
-    verify(bulkMessageWriter, times(1)).init(eq(stormConf), any(EnrichmentConfigurations.class));
+    verify(bulkMessageWriter, times(1)).init(eq(stormConf), any(WriterConfiguration.class));
     tupleList = new ArrayList<>();
     for(int i = 0; i < 4; i++) {
       when(tuple.getValueByField("message")).thenReturn(messageList.get(i));
       tupleList.add(tuple);
       bulkMessageWriterBolt.execute(tuple);
-      verify(bulkMessageWriter, times(0)).write(eq(sensorType), any(EnrichmentConfigurations.class), eq(tupleList), eq(messageList));
+      verify(bulkMessageWriter, times(0)).write(eq(sensorType), any(WriterConfiguration.class), eq(tupleList), eq(messageList));
     }
     when(tuple.getValueByField("message")).thenReturn(messageList.get(4));
     tupleList.add(tuple);
     bulkMessageWriterBolt.execute(tuple);
-    verify(bulkMessageWriter, times(1)).write(eq(sensorType), any(EnrichmentConfigurations.class), eq(tupleList), argThat(new MessageListMatcher(messageList)));
+    verify(bulkMessageWriter, times(1)).write(eq(sensorType), any(WriterConfiguration.class), eq(tupleList), argThat(new MessageListMatcher(messageList)));
     verify(outputCollector, times(5)).ack(tuple);
     reset(outputCollector);
-    doThrow(new Exception()).when(bulkMessageWriter).write(eq(sensorType), any(EnrichmentConfigurations.class), Matchers.anyListOf(Tuple.class), Matchers.anyListOf(JSONObject.class));
+    doThrow(new Exception()).when(bulkMessageWriter).write(eq(sensorType), any(WriterConfiguration.class), Matchers.anyListOf(Tuple.class), Matchers.anyListOf(JSONObject.class));
     when(tuple.getValueByField("message")).thenReturn(messageList.get(0));
     for(int i = 0; i < 5; i++) {
       bulkMessageWriterBolt.execute(tuple);

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/converter/EnrichmentConverterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/converter/EnrichmentConverterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/converter/EnrichmentConverterTest.java
index 20ec64c..e506c97 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/converter/EnrichmentConverterTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/converter/EnrichmentConverterTest.java
@@ -39,7 +39,7 @@ public class EnrichmentConverterTest {
   public void testValueConversion() throws IOException {
     EnrichmentConverter converter = new EnrichmentConverter();
     EnrichmentKey k1 = new EnrichmentKey("type", "indicator");
-    EnrichmentValue v1 = new EnrichmentValue(new HashMap<String, String>() {{
+    EnrichmentValue v1 = new EnrichmentValue(new HashMap<String, Object>() {{
       put("k1", "v1");
       put("k2", "v2");
     }});

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java
index 1fd69b3..e4d48dc 100644
--- a/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java
+++ b/metron-platform/metron-hbase/src/main/java/org/apache/metron/hbase/writer/HBaseWriter.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
 import org.apache.metron.hbase.HTableProvider;
 import org.apache.metron.hbase.TableProvider;
 import org.apache.metron.common.utils.ReflectionUtils;
@@ -62,7 +63,7 @@ public abstract class HBaseWriter implements MessageWriter<JSONObject>, Serializ
   }
 
   @Override
-  public void write(String sourceType, Configurations configurations, Tuple tuple, JSONObject message) throws Exception {
+  public void write(String sourceType, WriterConfiguration configurations, Tuple tuple, JSONObject message) throws Exception {
     Put put = new Put(getKey(tuple, message));
     Map<String, byte[]> values = getValues(tuple, message);
     for(String column: values.keySet()) {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
index 521ffdf..483f2b5 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
@@ -65,7 +65,7 @@ public abstract class EnrichmentIntegrationTest extends BaseIntegrationTest {
   private static final String DST_IP = "ip_dst_addr";
   private static final String MALICIOUS_IP_TYPE = "malicious_ip";
   private static final String PLAYFUL_CLASSIFICATION_TYPE = "playful_classification";
-  private static final Map<String, String> PLAYFUL_ENRICHMENT = new HashMap<String, String>() {{
+  private static final Map<String, Object> PLAYFUL_ENRICHMENT = new HashMap<String, Object>() {{
     put("orientation", "north");
   }};
   protected String testSensorType = "test";
@@ -183,7 +183,7 @@ public abstract class EnrichmentIntegrationTest extends BaseIntegrationTest {
     final MockHTable trackerTable = (MockHTable)MockHTable.Provider.addToCache(trackerHBaseTableName, cf);
     final MockHTable threatIntelTable = (MockHTable)MockHTable.Provider.addToCache(threatIntelTableName, cf);
     EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>(){{
-      add(new LookupKV<>(new EnrichmentKey(MALICIOUS_IP_TYPE, "10.0.2.3"), new EnrichmentValue(new HashMap<String, String>())));
+      add(new LookupKV<>(new EnrichmentKey(MALICIOUS_IP_TYPE, "10.0.2.3"), new EnrichmentValue(new HashMap<>())));
     }});
     final MockHTable enrichmentTable = (MockHTable)MockHTable.Provider.addToCache(enrichmentsTableName, cf);
     EnrichmentHelper.INSTANCE.load(enrichmentTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>(){{
@@ -206,7 +206,7 @@ public abstract class EnrichmentIntegrationTest extends BaseIntegrationTest {
             .withComponent("config", configUploadComponent)
             .withComponent("search", searchComponent)
             .withComponent("storm", fluxComponent)
-            .withMillisecondsBetweenAttempts(10000)
+            .withMillisecondsBetweenAttempts(15000)
             .withNumRetries(10)
             .build();
     runner.start();

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
index 0a088cb..865b017 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
@@ -23,6 +23,8 @@ import org.apache.metron.common.configuration.SensorParserConfig;
 import org.apache.metron.integration.InMemoryComponent;
 import org.apache.metron.integration.UnableToStartException;
 
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Properties;
 
 public class ConfigUploadComponent implements InMemoryComponent {
@@ -31,7 +33,7 @@ public class ConfigUploadComponent implements InMemoryComponent {
   private String globalConfigPath;
   private String parserConfigsPath;
   private String enrichmentConfigsPath;
-
+  private Map<String, SensorParserConfig> parserSensorConfigs = new HashMap<>();
   public ConfigUploadComponent withTopologyProperties(Properties topologyProperties) {
     this.topologyProperties = topologyProperties;
     return this;
@@ -51,11 +53,27 @@ public class ConfigUploadComponent implements InMemoryComponent {
     return this;
   }
 
+  public ConfigUploadComponent withParserSensorConfig(String name, SensorParserConfig config) {
+    parserSensorConfigs.put(name, config);
+    return this;
+  }
+
 
   @Override
   public void start() throws UnableToStartException {
     try {
-      ConfigurationsUtils.uploadConfigsToZookeeper(globalConfigPath, parserConfigsPath, enrichmentConfigsPath, topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY));
+      ConfigurationsUtils.uploadConfigsToZookeeper( globalConfigPath
+                                                  , parserConfigsPath
+                                                  , enrichmentConfigsPath
+                                                  , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
+                                                  );
+      for(Map.Entry<String, SensorParserConfig> kv : parserSensorConfigs.entrySet()) {
+        ConfigurationsUtils.writeSensorParserConfigToZookeeper( kv.getKey()
+                                                              , kv.getValue()
+                                                              , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
+                                                              );
+      }
+
     } catch (Exception e) {
       throw new UnableToStartException(e.getMessage(), e);
     }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java
index f991fe2..99f916d 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/KafkaWithZKComponent.java
@@ -136,6 +136,7 @@ public class KafkaWithZKComponent implements InMemoryComponent {
 
     // setup Broker
     Properties props = TestUtils.createBrokerConfig(0, brokerPort, true);
+    props.setProperty("zookeeper.connection.timeout.ms","1000000");
     KafkaConfig config = new KafkaConfig(props);
     Time mock = new MockTime();
     kafkaServer = TestUtils.createServer(config, mock);

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java
new file mode 100644
index 0000000..becadce
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.integration.mock;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.test.mock.MockHTable;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+public class MockTableProvider implements TableProvider, Serializable {
+  static MockHTable.Provider provider = new MockHTable.Provider();
+  @Override
+  public HTableInterface getTable(Configuration config, String tableName) throws IOException {
+    return provider.getTable(config, tableName);
+  }
+  public static void addTable(String tableName, String... cf) {
+    provider.addToCache(tableName, cf);
+  }
+  public static MockHTable getTable(String tableName) {
+    try {
+      return (MockHTable) provider.getTable(null, tableName);
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to get table: " + tableName);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index b42c2ed..9efcd56 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -32,6 +32,16 @@
             <version>${project.parent.version}</version>
         </dependency>
         <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-enrichment</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-pcap</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${global_hadoop_version}</version>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
index addeaab..b68bac5 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
@@ -20,9 +20,17 @@ package org.apache.metron.parsers.bolt;
 import backtype.storm.task.OutputCollector;
 import backtype.storm.task.TopologyContext;
 import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
 import backtype.storm.tuple.Tuple;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.bolt.ConfiguredParserBolt;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
+import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFacade;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.writer.BulkWriterComponent;
+import org.apache.metron.common.writer.WriterToBulkWriter;
 import org.apache.metron.common.configuration.SensorParserConfig;
 import org.apache.metron.parsers.filters.Filters;
 import org.apache.metron.common.configuration.FieldTransformer;
@@ -34,20 +42,46 @@ import org.apache.metron.parsers.interfaces.MessageParser;
 import org.apache.metron.common.interfaces.MessageWriter;
 import org.json.simple.JSONObject;
 
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
-public class ParserBolt extends ConfiguredParserBolt {
+public class ParserBolt extends ConfiguredParserBolt implements Serializable {
 
   private OutputCollector collector;
   private MessageParser<JSONObject> parser;
-  private MessageFilter<JSONObject> filter;
-  private MessageWriter<JSONObject> writer;
+  private MessageFilter<JSONObject> filter = new GenericMessageFilter();
+  private transient Function<ParserConfigurations, WriterConfiguration> writerTransformer;
+  private BulkMessageWriter<JSONObject> messageWriter;
+  private BulkWriterComponent<JSONObject> writerComponent;
+  private boolean isBulk = false;
+  public ParserBolt( String zookeeperUrl
+                   , String sensorType
+                   , MessageParser<JSONObject> parser
+                   , MessageWriter<JSONObject> writer
+  )
+  {
+    super(zookeeperUrl, sensorType);
+    isBulk = false;
+    this.parser = parser;
+    messageWriter = new WriterToBulkWriter<>(writer);
+  }
 
-  public ParserBolt(String zookeeperUrl, String sensorType, MessageParser<JSONObject> parser, MessageWriter<JSONObject> writer) {
+  public ParserBolt( String zookeeperUrl
+                   , String sensorType
+                   , MessageParser<JSONObject> parser
+                   , BulkMessageWriter<JSONObject> writer
+  )
+  {
     super(zookeeperUrl, sensorType);
+    isBulk = true;
     this.parser = parser;
-    this.writer = writer;
+    messageWriter = writer;
+
+
   }
 
   public ParserBolt withMessageFilter(MessageFilter<JSONObject> filter) {
@@ -69,7 +103,24 @@ public class ParserBolt extends ConfiguredParserBolt {
       );
     }
     parser.init();
-    writer.init();
+
+    if(isBulk) {
+      writerTransformer = config -> new ParserWriterConfiguration(config);
+    }
+    else {
+      writerTransformer = config -> new SingleBatchConfigurationFacade(new ParserWriterConfiguration(config));
+    }
+    try {
+      messageWriter.init(stormConf, writerTransformer.apply(getConfigurations()));
+    } catch (Exception e) {
+      throw new IllegalStateException("Unable to initialize message writer", e);
+    }
+    this.writerComponent = new BulkWriterComponent<JSONObject>(collector, isBulk, isBulk) {
+      @Override
+      protected Collection<Tuple> createTupleCollection() {
+        return new HashSet<>();
+      }
+    };
     SensorParserConfig config = getSensorParserConfig();
     if(config != null) {
       config.init();
@@ -77,6 +128,7 @@ public class ParserBolt extends ConfiguredParserBolt {
     else {
       throw new IllegalStateException("Unable to retrieve a parser config for " + getSensorType());
     }
+    parser.configure(config.getParserConfig());
   }
 
 
@@ -86,23 +138,27 @@ public class ParserBolt extends ConfiguredParserBolt {
     byte[] originalMessage = tuple.getBinary(0);
     SensorParserConfig sensorParserConfig = getSensorParserConfig();
     try {
+      boolean ackTuple = true;
       if(sensorParserConfig != null) {
         List<JSONObject> messages = parser.parse(originalMessage);
         for (JSONObject message : messages) {
           if (parser.validate(message)) {
             if (filter != null && filter.emitTuple(message)) {
+              ackTuple = !isBulk;
               message.put(Constants.SENSOR_TYPE, getSensorType());
               for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) {
                 if (handler != null) {
                   handler.transformAndUpdate(message, sensorParserConfig.getParserConfig());
                 }
               }
-              writer.write(getSensorType(), configurations, tuple, message);
+              writerComponent.write(getSensorType(), tuple, message, messageWriter, writerTransformer.apply(getConfigurations()));
             }
           }
         }
       }
-      collector.ack(tuple);
+      if(ackTuple) {
+        collector.ack(tuple);
+      }
     } catch (Throwable ex) {
       ErrorUtils.handleError(collector, ex, Constants.ERROR_STREAM);
       collector.fail(tuple);
@@ -111,6 +167,6 @@ public class ParserBolt extends ConfiguredParserBolt {
 
   @Override
   public void declareOutputFields(OutputFieldsDeclarer declarer) {
-
+    declarer.declareStream(Constants.ERROR_STREAM, new Fields("message"));
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java
new file mode 100644
index 0000000..a0097ed
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/csv/CSVParser.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.csv;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.metron.common.csv.CSVConverter;
+import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.SimpleDateFormat;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class CSVParser extends BasicParser {
+  protected static final Logger LOG = LoggerFactory.getLogger(CSVParser.class);
+  public static final String TIMESTAMP_FORMAT_CONF = "timestampFormat";
+  private transient CSVConverter converter;
+  private SimpleDateFormat timestampFormat;
+  @Override
+  public void configure(Map<String, Object> parserConfig) {
+    converter = new CSVConverter();
+    converter.initialize(parserConfig);
+    Object tsFormatObj = parserConfig.get(TIMESTAMP_FORMAT_CONF);
+    if(tsFormatObj != null) {
+      timestampFormat = new SimpleDateFormat(tsFormatObj.toString());
+    }
+  }
+
+  @Override
+  public void init() {
+
+  }
+
+
+  @Override
+  public List<JSONObject> parse(byte[] rawMessage) {
+    try {
+      String msg = new String(rawMessage, "UTF-8");
+      Map<String, String> value = converter.toMap(msg);
+      if(value != null) {
+        value.put("original_string", msg);
+        Object timestampObj = value.get("timestamp");
+        Long timestamp = null;
+        if(timestampObj == null) {
+          timestamp = System.currentTimeMillis();
+        }
+        else {
+          if(timestampFormat == null) {
+            timestamp = ConversionUtils.convert(timestampObj, Long.class);
+          }
+          else {
+            try {
+              timestamp = timestampFormat.parse(timestampObj.toString()).getTime();
+            }
+            catch(Exception e) {
+              LOG.error("Unable to format " + timestampObj.toString());
+            }
+          }
+        }
+        JSONObject jsonVal = new JSONObject(value);
+        jsonVal.put("timestamp", timestamp);
+        return ImmutableList.of(jsonVal);
+      }
+      else {
+        return Collections.emptyList();
+      }
+    } catch (Exception e) {
+      LOG.error("Unable to parse " + new String(rawMessage), e);
+      return Collections.emptyList();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
index 598de55..ad6068f 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
@@ -20,9 +20,14 @@ package org.apache.metron.parsers.topology;
 import backtype.storm.topology.TopologyBuilder;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.configuration.ParserConfigurations;
 import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.interfaces.MessageWriter;
 import org.apache.metron.common.spout.kafka.SpoutConfig;
 import org.apache.metron.common.utils.ReflectionUtils;
+import org.apache.metron.common.writer.AbstractWriter;
 import org.apache.metron.parsers.bolt.ParserBolt;
 import org.apache.metron.parsers.interfaces.MessageParser;
 import org.apache.metron.parsers.writer.KafkaWriter;
@@ -45,7 +50,13 @@ public class ParserTopologyBuilder {
                                      ) throws Exception {
     CuratorFramework client = ConfigurationsUtils.getClient(zookeeperUrl);
     client.start();
-    SensorParserConfig sensorParserConfig = ConfigurationsUtils.readSensorParserConfigFromZookeeper(sensorType, client);
+    ParserConfigurations configurations = new ParserConfigurations();
+    ConfigurationsUtils.updateParserConfigsFromZookeeper(configurations, client);
+    SensorParserConfig sensorParserConfig = configurations.getSensorParserConfig(sensorType);
+    if(sensorParserConfig == null) {
+      throw new IllegalStateException("Cannot find the parser configuration in zookeeper for " + sensorType + "." +
+              "  Please check that it exists in zookeeper by using the zk_load_configs.sh -m DUMP command.");
+    }
     client.close();
     String sensorTopic = sensorParserConfig.getSensorTopic() != null ? sensorParserConfig.getSensorTopic() : sensorType;
     TopologyBuilder builder = new TopologyBuilder();
@@ -56,11 +67,31 @@ public class ParserTopologyBuilder {
            .setNumTasks(parserNumTasks);
     MessageParser<JSONObject> parser = ReflectionUtils.createInstance(sensorParserConfig.getParserClassName());
     parser.configure(sensorParserConfig.getParserConfig());
-    KafkaWriter writer = new KafkaWriter(brokerUrl);
-    ParserBolt parserBolt = new ParserBolt(zookeeperUrl, sensorType, parser, writer);
+    ParserBolt parserBolt = null;
+    {
+      if(sensorParserConfig.getWriterClassName() == null) {
+        KafkaWriter writer = new KafkaWriter(brokerUrl);
+        writer.configure(sensorType, new ParserWriterConfiguration(configurations));
+        parserBolt = new ParserBolt(zookeeperUrl, sensorType, parser, writer);
+      }
+      else {
+        AbstractWriter writer = ReflectionUtils.createInstance(sensorParserConfig.getWriterClassName());
+        writer.configure(sensorType, new ParserWriterConfiguration(configurations));
+        if(writer instanceof BulkMessageWriter) {
+          parserBolt = new ParserBolt(zookeeperUrl, sensorType, parser, (BulkMessageWriter<JSONObject>)writer);
+        }
+        else if(writer instanceof MessageWriter) {
+          parserBolt = new ParserBolt(zookeeperUrl, sensorType, parser, (MessageWriter<JSONObject>)writer);
+        }
+        else {
+          throw new IllegalStateException("Unable to create parser bolt: writer must be a MessageWriter or a BulkMessageWriter");
+        }
+      }
+    }
     builder.setBolt("parserBolt", parserBolt, parserParallelism)
            .setNumTasks(spoutNumTasks)
            .shuffleGrouping("kafkaSpout");
     return builder;
   }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
index d168e29..6262dc1 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
@@ -22,21 +22,50 @@ import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.ProducerRecord;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
 import org.apache.metron.common.interfaces.MessageWriter;
+import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.common.writer.AbstractWriter;
 import org.json.simple.JSONObject;
 
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
 
-public class KafkaWriter implements MessageWriter<JSONObject>, Serializable {
-
+public class KafkaWriter extends AbstractWriter implements MessageWriter<JSONObject>, Serializable {
+  public enum Configurations {
+     BROKER("kafka.brokerUrl")
+    ,KEY_SERIALIZER("kafka.keySerializer")
+    ,VALUE_SERIALIZER("kafka.valueSerializer")
+    ,REQUIRED_ACKS("kafka.requiredAcks")
+    ,TOPIC("kafka.topic");
+    ;
+    String key;
+    Configurations(String key) {
+      this.key = key;
+    }
+    public Object get(Map<String, Object> config) {
+      return config.get(key);
+    }
+    public <T> T getAndConvert(Map<String, Object> config, Class<T> clazz) {
+      Object o = get(config);
+      if(o != null) {
+        return ConversionUtils.convert(o, clazz);
+      }
+      return null;
+    }
+  }
   private String brokerUrl;
   private String keySerializer = "org.apache.kafka.common.serialization.StringSerializer";
   private String valueSerializer = "org.apache.kafka.common.serialization.StringSerializer";
   private int requiredAcks = 1;
+  private String kafkaTopic = Constants.ENRICHMENT_TOPIC;
   private KafkaProducer kafkaProducer;
 
+  public KafkaWriter() {}
+
+
+
   public KafkaWriter(String brokerUrl) {
     this.brokerUrl = brokerUrl;
   }
@@ -51,11 +80,40 @@ public class KafkaWriter implements MessageWriter<JSONObject>, Serializable {
     return this;
   }
 
-  public KafkaWriter withRequiredAcks(int requiredAcks) {
+  public KafkaWriter withRequiredAcks(Integer requiredAcks) {
     this.requiredAcks = requiredAcks;
     return this;
   }
 
+  public KafkaWriter withTopic(String topic) {
+    this.kafkaTopic= topic;
+    return this;
+  }
+  @Override
+  public void configure(String sensorName, WriterConfiguration configuration) {
+    Map<String, Object> configMap = configuration.getSensorConfig(sensorName);
+    String brokerUrl = Configurations.BROKER.getAndConvert(configMap, String.class);
+    if(brokerUrl != null) {
+      this.brokerUrl = brokerUrl;
+    }
+    String keySerializer = Configurations.KEY_SERIALIZER.getAndConvert(configMap, String.class);
+    if(keySerializer != null) {
+      withKeySerializer(keySerializer);
+    }
+    String valueSerializer = Configurations.VALUE_SERIALIZER.getAndConvert(configMap, String.class);
+    if(valueSerializer != null) {
+      withValueSerializer(keySerializer);
+    }
+    Integer requiredAcks = Configurations.REQUIRED_ACKS.getAndConvert(configMap, Integer.class);
+    if(requiredAcks!= null) {
+      withRequiredAcks(requiredAcks);
+    }
+    String topic = Configurations.TOPIC.getAndConvert(configMap, String.class);
+    if(topic != null) {
+      withTopic(topic);
+    }
+  }
+
   @Override
   public void init() {
     Map<String, Object> producerConfig = new HashMap<>();
@@ -68,8 +126,8 @@ public class KafkaWriter implements MessageWriter<JSONObject>, Serializable {
 
   @SuppressWarnings("unchecked")
   @Override
-  public void write(String sourceType, Configurations configurations, Tuple tuple, JSONObject message) throws Exception {
-    kafkaProducer.send(new ProducerRecord<String, String>(Constants.ENRICHMENT_TOPIC, message.toJSONString()));
+  public void write(String sourceType, WriterConfiguration configurations, Tuple tuple, JSONObject message) throws Exception {
+    kafkaProducer.send(new ProducerRecord<String, String>(kafkaTopic, message.toJSONString()));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
index 658d52e..73aad23 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
@@ -17,6 +17,16 @@
  */
 package org.apache.metron.parsers.bolt;
 
+import org.apache.metron.common.configuration.SensorParserConfig;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.tuple.Tuple;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.metron.common.configuration.ParserConfigurations;
@@ -28,6 +38,7 @@ import org.apache.metron.parsers.interfaces.MessageParser;
 import org.apache.metron.common.interfaces.MessageWriter;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
+import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.Mock;
 
@@ -35,6 +46,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
@@ -52,18 +64,42 @@ public class ParserBoltTest extends BaseBoltTest {
   private MessageWriter<JSONObject> writer;
 
   @Mock
+  private BulkMessageWriter<JSONObject> batchWriter;
+
+  @Mock
   private MessageFilter<JSONObject> filter;
 
+  @Mock
+  private Tuple t1;
+
+  @Mock
+  private Tuple t2;
+
+  @Mock
+  private Tuple t3;
+
+  @Mock
+  private Tuple t4;
+
+  @Mock
+  private Tuple t5;
+
   @Test
   public void test() throws Exception {
     String sensorType = "yaf";
     ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, writer) {
       @Override
-      public ParserConfigurations getConfigurations() {
+      protected ParserConfigurations defaultConfigurations() {
         return new ParserConfigurations() {
           @Override
           public SensorParserConfig getSensorParserConfig(String sensorType) {
-            return new SensorParserConfig();
+            return new SensorParserConfig() {
+              @Override
+              public Map<String, Object> getParserConfig() {
+                return new HashMap<String, Object>() {{
+                }};
+              }
+            };
           }
         };
       }
@@ -88,7 +124,7 @@ public class ParserBoltTest extends BaseBoltTest {
     when(parser.validate(eq(messages.get(0)))).thenReturn(true);
     when(parser.validate(eq(messages.get(1)))).thenReturn(false);
     parserBolt.execute(tuple);
-    verify(writer, times(1)).write(eq(sensorType), any(Configurations.class), eq(tuple), eq(finalMessage1));
+    verify(writer, times(1)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage1));
     verify(outputCollector, times(1)).ack(tuple);
     when(parser.validate(eq(messages.get(0)))).thenReturn(true);
     when(parser.validate(eq(messages.get(1)))).thenReturn(true);
@@ -96,12 +132,46 @@ public class ParserBoltTest extends BaseBoltTest {
     when(filter.emitTuple(messages.get(1))).thenReturn(true);
     parserBolt.withMessageFilter(filter);
     parserBolt.execute(tuple);
-    verify(writer, times(1)).write(eq(sensorType), any(Configurations.class), eq(tuple), eq(finalMessage2));
+    verify(writer, times(1)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage2));
     verify(outputCollector, times(2)).ack(tuple);
-    doThrow(new Exception()).when(writer).write(eq(sensorType), any(Configurations.class), eq(tuple), eq(finalMessage2));
+    doThrow(new Exception()).when(writer).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage2));
     parserBolt.execute(tuple);
     verify(outputCollector, times(1)).reportError(any(Throwable.class));
   }
+@Test
+public void testImplicitBatchOfOne() throws Exception {
+
+  String sensorType = "yaf";
+
+  ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, batchWriter) {
+    @Override
+    protected ParserConfigurations defaultConfigurations() {
+      return new ParserConfigurations() {
+        @Override
+        public SensorParserConfig getSensorParserConfig(String sensorType) {
+          return new SensorParserConfig() {
+            @Override
+            public Map<String, Object> getParserConfig() {
+              return new HashMap<String, Object>() {{
+              }};
+            }
+          };
+        }
+      };
+    }
+  };
+  parserBolt.setCuratorFramework(client);
+  parserBolt.setTreeCache(cache);
+  parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
+  verify(parser, times(1)).init();
+  verify(batchWriter, times(1)).init(any(), any());
+  when(parser.validate(any())).thenReturn(true);
+  when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject()));
+  when(filter.emitTuple(any())).thenReturn(true);
+  parserBolt.withMessageFilter(filter);
+  parserBolt.execute(t1);
+  verify(outputCollector, times(1)).ack(t1);
+}
 
   /**
    {
@@ -116,7 +186,7 @@ public class ParserBoltTest extends BaseBoltTest {
   @Test
   public void testFilter() throws Exception {
     String sensorType = "yaf";
-    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, writer) {
+    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, batchWriter) {
       @Override
       protected SensorParserConfig getSensorParserConfig() {
         try {
@@ -130,6 +200,150 @@ public class ParserBoltTest extends BaseBoltTest {
     parserBolt.setTreeCache(cache);
     parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
     verify(parser, times(1)).init();
+    verify(batchWriter, times(1)).init(any(), any());
+    when(parser.validate(any())).thenReturn(true);
+    when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject()));
+    parserBolt.withMessageFilter(filter);
+    parserBolt.execute(t1);
+    verify(outputCollector, times(1)).ack(t1);
+  }
+  @Test
+  public void testBatchOfOne() throws Exception {
+
+    String sensorType = "yaf";
+
+    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, batchWriter) {
+      @Override
+      protected ParserConfigurations defaultConfigurations() {
+        return new ParserConfigurations() {
+          @Override
+          public SensorParserConfig getSensorParserConfig(String sensorType) {
+            return new SensorParserConfig() {
+              @Override
+              public Map<String, Object> getParserConfig() {
+                return new HashMap<String, Object>() {{
+                  put(ParserWriterConfiguration.BATCH_CONF, "1");
+                }};
+              }
+            };
+          }
+        };
+      }
+    };
+    parserBolt.setCuratorFramework(client);
+    parserBolt.setTreeCache(cache);
+    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
+    verify(parser, times(1)).init();
+    verify(batchWriter, times(1)).init(any(), any());
+    when(parser.validate(any())).thenReturn(true);
+    when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject()));
+    when(filter.emitTuple(any())).thenReturn(true);
+    parserBolt.withMessageFilter(filter);
+    parserBolt.execute(t1);
+    verify(outputCollector, times(1)).ack(t1);
+  }
+  @Test
+  public void testBatchOfFive() throws Exception {
+
+    String sensorType = "yaf";
+
+    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, batchWriter) {
+      @Override
+      protected ParserConfigurations defaultConfigurations() {
+        return new ParserConfigurations() {
+          @Override
+          public SensorParserConfig getSensorParserConfig(String sensorType) {
+            return new SensorParserConfig() {
+              @Override
+              public Map<String, Object> getParserConfig() {
+                return new HashMap<String, Object>() {{
+                  put(ParserWriterConfiguration.BATCH_CONF, 5);
+                }};
+              }
+            };
+          }
+        };
+      }
+    };
+    parserBolt.setCuratorFramework(client);
+    parserBolt.setTreeCache(cache);
+    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
+    verify(parser, times(1)).init();
+    verify(batchWriter, times(1)).init(any(), any());
+    when(parser.validate(any())).thenReturn(true);
+    when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject()));
+    when(filter.emitTuple(any())).thenReturn(true);
+    parserBolt.withMessageFilter(filter);
+    writeNonBatch(outputCollector, parserBolt, t1);
+    writeNonBatch(outputCollector, parserBolt, t2);
+    writeNonBatch(outputCollector, parserBolt, t3);
+    writeNonBatch(outputCollector, parserBolt, t4);
+    parserBolt.execute(t5);
+    verify(outputCollector, times(1)).ack(t1);
+    verify(outputCollector, times(1)).ack(t2);
+    verify(outputCollector, times(1)).ack(t3);
+    verify(outputCollector, times(1)).ack(t4);
+    verify(outputCollector, times(1)).ack(t5);
+
+
+  }
+  @Test
+  public void testBatchOfFiveWithError() throws Exception {
+
+    String sensorType = "yaf";
+    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, batchWriter) {
+      @Override
+      protected ParserConfigurations defaultConfigurations() {
+        return new ParserConfigurations() {
+          @Override
+          public SensorParserConfig getSensorParserConfig(String sensorType) {
+            return new SensorParserConfig() {
+              @Override
+              public Map<String, Object> getParserConfig() {
+                return new HashMap<String, Object>() {{
+                  put(ParserWriterConfiguration.BATCH_CONF, 5);
+                }};
+              }
+            };
+          }
+        };
+      }
+    };
+    parserBolt.setCuratorFramework(client);
+    parserBolt.setTreeCache(cache);
+    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
+    verify(parser, times(1)).init();
+    verify(batchWriter, times(1)).init(any(), any());
+
+    doThrow(new Exception()).when(batchWriter).write(any(), any(), any(), any());
+    when(parser.validate(any())).thenReturn(true);
+    when(parser.parse(any())).thenReturn(ImmutableList.of(new JSONObject()));
+    when(filter.emitTuple(any())).thenReturn(true);
+    parserBolt.withMessageFilter(filter);
+    writeNonBatch(outputCollector, parserBolt, t1);
+    writeNonBatch(outputCollector, parserBolt, t2);
+    writeNonBatch(outputCollector, parserBolt, t3);
+    writeNonBatch(outputCollector, parserBolt, t4);
+    parserBolt.execute(t5);
+    verify(outputCollector, times(0)).ack(t1);
+    verify(outputCollector, times(1)).fail(t1);
+    verify(outputCollector, times(0)).ack(t2);
+    verify(outputCollector, times(1)).fail(t2);
+    verify(outputCollector, times(0)).ack(t3);
+    verify(outputCollector, times(1)).fail(t3);
+    verify(outputCollector, times(0)).ack(t4);
+    verify(outputCollector, times(1)).fail(t4);
+    verify(outputCollector, times(0)).ack(t5);
+    verify(outputCollector, times(1)).fail(t5);
+
+
+  }
+  private static void writeNonBatch(OutputCollector collector, ParserBolt bolt, Tuple t) {
+    bolt.execute(t);
+    verify(collector, times(0)).ack(t);
+  }
+
+/*=======
     verify(writer, times(1)).init();
     byte[] sampleBinary = "some binary message".getBytes();
     JSONParser jsonParser = new JSONParser();
@@ -147,4 +361,5 @@ public class ParserBoltTest extends BaseBoltTest {
     verify(writer, times(1)).write(eq(sensorType), any(Configurations.class), eq(tuple), eq(finalMessage1));
     verify(outputCollector, times(1)).ack(tuple);
   }
+>>>>>>> master*/
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java
new file mode 100644
index 0000000..5f314fa
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/csv/CSVParserTest.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.csv;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.utils.JSONUtils;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+public class CSVParserTest {
+  /**
+   {
+    "parserClassName" : "org.apache.metron.parsers.csv.CSVParser"
+   ,"sensorTopic":"dummy"
+   ,"parserConfig":
+   {
+    "columns" : {
+                "col1" : 0
+               ,"col2" : 1
+               ,"col3" : 2
+                 }
+   }
+   }
+   */
+  @Multiline
+  public static String parserConfig;
+
+  @Test
+  public void test() throws IOException {
+    CSVParser parser = new CSVParser();
+
+    SensorParserConfig config = JSONUtils.INSTANCE.load(parserConfig, SensorParserConfig.class);
+    parser.init();
+    parser.configure(config.getParserConfig());
+    {
+      String line = "#foo,bar,grok";
+      Assert.assertEquals(0, parser.parse(Bytes.toBytes(line)).size());
+    }
+    {
+      String line = "";
+      Assert.assertEquals(0, parser.parse(Bytes.toBytes(line)).size());
+    }
+    {
+      String line = "foo,bar,grok";
+      List<JSONObject> results = parser.parse(Bytes.toBytes(line));
+      Assert.assertEquals(1, results.size());
+      JSONObject o = results.get(0);
+      Assert.assertTrue(parser.validate(o));
+      Assert.assertEquals(5, o.size());
+      Assert.assertEquals("foo", o.get("col1"));
+      Assert.assertEquals("bar", o.get("col2"));
+      Assert.assertEquals("grok", o.get("col3"));
+    }
+    {
+      String line = "\"foo\", \"bar\",\"grok\"";
+      List<JSONObject> results = parser.parse(Bytes.toBytes(line));
+      Assert.assertEquals(1, results.size());
+      JSONObject o = results.get(0);
+      Assert.assertTrue(parser.validate(o));
+      Assert.assertEquals(5, o.size());
+      Assert.assertEquals("foo", o.get("col1"));
+      Assert.assertEquals("bar", o.get("col2"));
+      Assert.assertEquals("grok", o.get("col3"));
+    }
+    {
+      String line = "foo, bar, grok";
+      List<JSONObject> results = parser.parse(Bytes.toBytes(line));
+      Assert.assertEquals(1, results.size());
+      JSONObject o = results.get(0);
+      Assert.assertTrue(parser.validate(o));
+      Assert.assertEquals(5, o.size());
+      Assert.assertEquals("foo", o.get("col1"));
+      Assert.assertEquals(" bar", o.get("col2"));
+      Assert.assertEquals(" grok", o.get("col3"));
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
new file mode 100644
index 0000000..cea635f
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writers;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.enrichment.converter.EnrichmentConverter;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.lookup.LookupKV;
+import org.apache.metron.integration.mock.MockTableProvider;
+import org.apache.metron.test.mock.MockHTable;
+import org.apache.metron.writer.hbase.SimpleHbaseEnrichmentWriter;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class SimpleHBaseEnrichmentWriterTest {
+  private static final String SENSOR_TYPE= "dummy";
+  private static final String TABLE_NAME= SENSOR_TYPE;
+  private static final String TABLE_CF= "cf";
+  private static final String ENRICHMENT_TYPE = "et";
+  private static final Map<String, Object> BASE_WRITER_CONFIG = new HashMap<String, Object>() {{
+    put(SimpleHbaseEnrichmentWriter.Configurations.HBASE_TABLE.getKey(), TABLE_NAME);
+    put(SimpleHbaseEnrichmentWriter.Configurations.HBASE_CF.getKey(), TABLE_CF);
+    put(SimpleHbaseEnrichmentWriter.Configurations.ENRICHMENT_TYPE.getKey(), ENRICHMENT_TYPE);
+    put(SimpleHbaseEnrichmentWriter.Configurations.HBASE_PROVIDER.getKey(), MockTableProvider.class.getName());
+  }};
+  @Before
+  public void setupMockTable() {
+    MockTableProvider.addTable(TABLE_NAME, TABLE_CF);
+  }
+  @Test
+  public void testBatchOneNormalPath() throws Exception {
+    final String sensorType = "dummy";
+    SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
+
+    WriterConfiguration configuration = createConfig(1,
+            new HashMap<String, Object>(BASE_WRITER_CONFIG) {{
+              put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
+            }}
+    );
+    writer.configure(sensorType,configuration);
+
+    writer.write( SENSOR_TYPE
+            , configuration
+            , null
+            , new ArrayList<JSONObject>() {{
+              add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar")));
+            }}
+    );
+    List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues();
+    Assert.assertEquals(1, values.size());
+    Assert.assertEquals("localhost", values.get(0).getKey().indicator);
+    Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user"));
+    Assert.assertEquals("bar", values.get(0).getValue().getMetadata().get("foo"));
+    Assert.assertEquals(2, values.get(0).getValue().getMetadata().size());
+  }
+
+  @Test
+  public void testFilteredKey() throws Exception {
+    final String sensorType = "dummy";
+    SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
+
+    WriterConfiguration configuration = createConfig(1,
+            new HashMap<String, Object>(BASE_WRITER_CONFIG) {{
+              put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
+              put(SimpleHbaseEnrichmentWriter.Configurations.VALUE_COLUMNS.getKey(), "user");
+            }}
+    );
+    writer.configure(sensorType,configuration);
+
+    writer.write( SENSOR_TYPE
+            , configuration
+            , null
+            , new ArrayList<JSONObject>() {{
+              add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar")));
+            }}
+    );
+    List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues();
+    Assert.assertEquals(1, values.size());
+    Assert.assertEquals("localhost", values.get(0).getKey().indicator);
+    Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user"));
+    Assert.assertNull(values.get(0).getValue().getMetadata().get("foo"));
+    Assert.assertEquals(1, values.get(0).getValue().getMetadata().size());
+  }
+
+  @Test
+  public void testFilteredKeys() throws Exception {
+    final String sensorType = "dummy";
+    SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter();
+
+    WriterConfiguration configuration = createConfig(1,
+            new HashMap<String, Object>(BASE_WRITER_CONFIG) {{
+              put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip");
+              put(SimpleHbaseEnrichmentWriter.Configurations.VALUE_COLUMNS.getKey(), ImmutableList.of("user", "ip"));
+            }}
+    );
+    writer.configure(sensorType,configuration);
+
+    writer.write( SENSOR_TYPE
+            , configuration
+            , null
+            , new ArrayList<JSONObject>() {{
+              add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar")));
+            }}
+    );
+    List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues();
+    Assert.assertEquals(1, values.size());
+    Assert.assertEquals("localhost", values.get(0).getKey().indicator);
+    Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user"));
+    Assert.assertEquals("localhost", values.get(0).getValue().getMetadata().get("ip"));
+    Assert.assertNull(values.get(0).getValue().getMetadata().get("foo"));
+    Assert.assertEquals(2, values.get(0).getValue().getMetadata().size());
+  }
+  public static List<LookupKV<EnrichmentKey, EnrichmentValue>> getValues() throws IOException {
+    MockHTable table = MockTableProvider.getTable(TABLE_NAME);
+    Assert.assertNotNull(table);
+    List<LookupKV<EnrichmentKey, EnrichmentValue>> ret = new ArrayList<>();
+    EnrichmentConverter converter = new EnrichmentConverter();
+    for(Result r : table.getScanner(Bytes.toBytes(TABLE_CF))) {
+      ret.add(converter.fromResult(r, TABLE_CF));
+    }
+    return ret;
+  }
+  public static WriterConfiguration createConfig(final int batchSize, final Map<String, Object> sensorConfig)
+  {
+    return new WriterConfiguration() {
+      @Override
+      public int getBatchSize(String sensorName) {
+        return batchSize;
+      }
+
+      @Override
+      public String getIndex(String sensorName) {
+        return SENSOR_TYPE;
+      }
+
+      @Override
+      public Map<String, Object> getSensorConfig(String sensorName) {
+        return sensorConfig;
+
+      }
+
+      @Override
+      public Map<String, Object> getGlobalConfig() {
+        return null;
+      }
+    };
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/d3efe3fb/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
new file mode 100644
index 0000000..f6436b9
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writers.integration;
+
+import com.google.common.collect.ImmutableList;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.TestConstants;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.enrichment.converter.EnrichmentConverter;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.lookup.LookupKV;
+import org.apache.metron.integration.*;
+import org.apache.metron.integration.components.ConfigUploadComponent;
+import org.apache.metron.integration.components.KafkaWithZKComponent;
+import org.apache.metron.integration.mock.MockTableProvider;
+import org.apache.metron.integration.utils.TestUtils;
+import org.apache.metron.parsers.integration.components.ParserTopologyComponent;
+import org.apache.metron.test.TestDataType;
+import org.apache.metron.test.mock.MockHTable;
+import org.apache.metron.test.utils.SampleDataUtils;
+import org.apache.metron.test.utils.UnitTestHelper;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.*;
+
+public class SimpleHbaseEnrichmentWriterIntegrationTest extends BaseIntegrationTest {
+
+  /**
+   {
+    "parserClassName" : "org.apache.metron.parsers.csv.CSVParser"
+   ,"writerClassName" : "org.apache.metron.writer.hbase.SimpleHbaseEnrichmentWriter"
+   ,"sensorTopic":"dummy"
+   ,"parserConfig":
+   {
+     "shew.table" : "dummy"
+    ,"shew.cf" : "cf"
+    ,"shew.keyColumns" : "col2"
+    ,"shew.enrichmentType" : "et"
+    ,"shew.hbaseProvider" : "org.apache.metron.integration.mock.MockTableProvider"
+    ,"columns" : {
+                "col1" : 0
+               ,"col2" : 1
+               ,"col3" : 2
+                 }
+   }
+   }
+   */
+  @Multiline
+  public static String parserConfig;
+
+  @Test
+  public void test() throws UnableToStartException, IOException {
+    final String sensorType = "dummy";
+    final List<byte[]> inputMessages = new ArrayList<byte[]>() {{
+      add(Bytes.toBytes("col11,col12,col13"));
+      add(Bytes.toBytes("col21,col22,col23"));
+      add(Bytes.toBytes("col31,col32,col33"));
+    }};
+    MockTableProvider.addTable(sensorType, "cf");
+    final Properties topologyProperties = new Properties();
+    final KafkaWithZKComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaWithZKComponent.Topic>() {{
+      add(new KafkaWithZKComponent.Topic(sensorType, 1));
+    }});
+    topologyProperties.setProperty("kafka.broker", kafkaComponent.getBrokerList());
+
+    ConfigUploadComponent configUploadComponent = new ConfigUploadComponent()
+            .withTopologyProperties(topologyProperties)
+            .withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH)
+            .withParserSensorConfig(sensorType, JSONUtils.INSTANCE.load(parserConfig, SensorParserConfig.class));
+
+    ParserTopologyComponent parserTopologyComponent = new ParserTopologyComponent.Builder()
+            .withSensorType(sensorType)
+            .withTopologyProperties(topologyProperties)
+            .withBrokerUrl(kafkaComponent.getBrokerList()).build();
+
+    UnitTestHelper.verboseLogging();
+    ComponentRunner runner = new ComponentRunner.Builder()
+            .withComponent("kafka", kafkaComponent)
+            .withComponent("config", configUploadComponent)
+            .withComponent("storm", parserTopologyComponent)
+            .withMillisecondsBetweenAttempts(5000)
+            .withNumRetries(10)
+            .build();
+    try {
+      runner.start();
+      kafkaComponent.writeMessages(sensorType, inputMessages);
+      List<LookupKV<EnrichmentKey, EnrichmentValue>> outputMessages =
+              runner.process(new Processor<List<LookupKV<EnrichmentKey, EnrichmentValue>>>() {
+                List<LookupKV<EnrichmentKey, EnrichmentValue>> messages = null;
+
+                public ReadinessState process(ComponentRunner runner) {
+                  MockHTable table = MockTableProvider.getTable(sensorType);
+                  if (table != null && table.size() == inputMessages.size()) {
+                    EnrichmentConverter converter = new EnrichmentConverter();
+                    messages = new ArrayList<>();
+                    try {
+                      for (Result r : table.getScanner(Bytes.toBytes("cf"))) {
+                        messages.add(converter.fromResult(r, "cf"));
+                      }
+                    } catch (IOException e) {
+                    }
+                    return ReadinessState.READY;
+                  }
+                  return ReadinessState.NOT_READY;
+                }
+
+                public List<LookupKV<EnrichmentKey, EnrichmentValue>> getResult() {
+                  return messages;
+                }
+              });
+      Set<String> validIndicators = new HashSet<>(ImmutableList.of("col12", "col22", "col32"));
+      Map<String, Map<String, String>> validMetadata = new HashMap<String, Map<String, String>>() {{
+        put("col12", new HashMap<String, String>() {{
+          put("col1", "col11");
+          put("col3", "col13");
+        }});
+        put("col22", new HashMap<String, String>() {{
+          put("col1", "col21");
+          put("col3", "col23");
+        }});
+        put("col32", new HashMap<String, String>() {{
+          put("col1", "col31");
+          put("col3", "col33");
+        }});
+      }};
+      for (LookupKV<EnrichmentKey, EnrichmentValue> kv : outputMessages) {
+        Assert.assertTrue(validIndicators.contains(kv.getKey().indicator));
+        Assert.assertEquals(kv.getValue().getMetadata().get("source.type"), "dummy");
+        Assert.assertNotNull(kv.getValue().getMetadata().get("timestamp"));
+        Assert.assertNotNull(kv.getValue().getMetadata().get("original_string"));
+        Map<String, String> metadata = validMetadata.get(kv.getKey().indicator);
+        for (Map.Entry<String, String> x : metadata.entrySet()) {
+          Assert.assertEquals(kv.getValue().getMetadata().get(x.getKey()), x.getValue());
+        }
+        Assert.assertEquals(metadata.size() + 3, kv.getValue().getMetadata().size());
+      }
+    }
+    finally {
+      if(runner != null) {
+        runner.stop();
+      }
+    }
+  }
+}