You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/07/20 13:14:24 UTC

[1/5] incubator-metron git commit: METRON-154: Decouple enrichment and indexing closes apache/incubator-metron#192

Repository: incubator-metron
Updated Branches:
  refs/heads/master 1c568bf04 -> 5ffcef8d4


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
new file mode 100644
index 0000000..0225137
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
+import org.apache.hadoop.hdfs.util.MD5FileUtils;
+import org.apache.hadoop.io.MD5Hash;
+import org.apache.log4j.Logger;
+import org.apache.storm.hdfs.bolt.format.FileNameFormat;
+import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
+import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy;
+import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
+import org.apache.storm.hdfs.common.rotation.RotationAction;
+import org.apache.storm.hdfs.common.security.HdfsSecurityUtil;
+import org.json.simple.JSONObject;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.*;
+
+public class SourceHandler {
+  private static final Logger LOG = Logger.getLogger(SourceHandler.class);
+  List<RotationAction> rotationActions = new ArrayList<>();
+  FileRotationPolicy rotationPolicy;
+  SyncPolicy syncPolicy;
+  FileNameFormat fileNameFormat;
+  private long offset = 0;
+  private int rotation = 0;
+  private transient FSDataOutputStream out;
+  private transient Object writeLock;
+  protected transient Timer rotationTimer; // only used for TimedRotationPolicy
+  protected transient FileSystem fs;
+  protected transient Path currentFile;
+  public SourceHandler(List<RotationAction> rotationActions
+                      , FileRotationPolicy rotationPolicy
+                      , SyncPolicy syncPolicy
+                      , FileNameFormat fileNameFormat
+                      , Map config
+                      ) throws IOException {
+    this.rotationActions = rotationActions;
+    this.rotationPolicy = rotationPolicy;
+    this.syncPolicy = syncPolicy;
+    this.fileNameFormat = fileNameFormat;
+    initialize(config);
+  }
+
+  public void handle(List<JSONObject> messages) throws Exception{
+
+    for(JSONObject message : messages) {
+      byte[] bytes = (message.toJSONString() + "\n").getBytes();
+      synchronized (this.writeLock) {
+        out.write(bytes);
+        this.offset += bytes.length;
+
+        if (this.syncPolicy.mark(null, this.offset)) {
+          if (this.out instanceof HdfsDataOutputStream) {
+            ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
+          } else {
+            this.out.hsync();
+          }
+          this.syncPolicy.reset();
+        }
+      }
+
+      if (this.rotationPolicy.mark(null, this.offset)) {
+        rotateOutputFile(); // synchronized
+        this.offset = 0;
+        this.rotationPolicy.reset();
+      }
+    }
+  }
+
+  private void initialize(Map config) throws IOException {
+    this.writeLock = new Object();
+    Configuration hdfsConfig = new Configuration();
+    this.fs = FileSystem.get(new Configuration());
+    HdfsSecurityUtil.login(config, hdfsConfig);
+    this.currentFile = createOutputFile();
+    if(this.rotationPolicy instanceof TimedRotationPolicy){
+      long interval = ((TimedRotationPolicy)this.rotationPolicy).getInterval();
+      this.rotationTimer = new Timer(true);
+      TimerTask task = new TimerTask() {
+        @Override
+        public void run() {
+          try {
+            rotateOutputFile();
+          } catch(IOException e){
+            LOG.warn("IOException during scheduled file rotation.", e);
+          }
+        }
+      };
+      this.rotationTimer.scheduleAtFixedRate(task, interval, interval);
+    }
+  }
+
+  protected void rotateOutputFile() throws IOException {
+    LOG.info("Rotating output file...");
+    long start = System.currentTimeMillis();
+    synchronized (this.writeLock) {
+      closeOutputFile();
+      this.rotation++;
+
+      Path newFile = createOutputFile();
+      LOG.info("Performing " +  this.rotationActions.size() + " file rotation actions." );
+      for (RotationAction action : this.rotationActions) {
+        action.execute(this.fs, this.currentFile);
+      }
+      this.currentFile = newFile;
+    }
+    long time = System.currentTimeMillis() - start;
+    LOG.info("File rotation took " + time + " ms.");
+  }
+
+  private Path createOutputFile() throws IOException {
+    Path path = new Path(this.fileNameFormat.getPath(), this.fileNameFormat.getName(this.rotation, System.currentTimeMillis()));
+    if(fs.getScheme().equals("file")) {
+      //in the situation where we're running this in a local filesystem, flushing doesn't work.
+      fs.mkdirs(path.getParent());
+      this.out = new FSDataOutputStream(new FileOutputStream(path.toString()), null);
+    }
+    else {
+      this.out = this.fs.create(path);
+    }
+    return path;
+  }
+
+  private void closeOutputFile() throws IOException {
+    this.out.close();
+  }
+
+
+  public void close() {
+    try {
+      closeOutputFile();
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to close output file.", e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java
new file mode 100644
index 0000000..f6acf6f
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer.kafka;
+
+import backtype.storm.tuple.Tuple;
+import com.google.common.base.Joiner;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.MessageWriter;
+import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.common.utils.KafkaUtils;
+import org.apache.metron.common.utils.StringUtils;
+import org.apache.metron.writer.AbstractWriter;
+import org.json.simple.JSONObject;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+public class KafkaWriter extends AbstractWriter implements MessageWriter<JSONObject>, Serializable {
+  public enum Configurations {
+     BROKER("kafka.brokerUrl")
+    ,KEY_SERIALIZER("kafka.keySerializer")
+    ,ZK_QUORUM("kafka.zkQuorum")
+    ,VALUE_SERIALIZER("kafka.valueSerializer")
+    ,REQUIRED_ACKS("kafka.requiredAcks")
+    ,TOPIC("kafka.topic")
+    ,PRODUCER_CONFIGS("kafka.producerConfigs");
+    ;
+    String key;
+    Configurations(String key) {
+      this.key = key;
+    }
+    public Object get(Optional<String> configPrefix, Map<String, Object> config) {
+      return config.get(StringUtils.join(".", configPrefix, Optional.of(key)));
+    }
+    public <T> T getAndConvert(Optional<String> configPrefix, Map<String, Object> config, Class<T> clazz) {
+      Object o = get(configPrefix, config);
+      if(o != null) {
+        return ConversionUtils.convert(o, clazz);
+      }
+      return null;
+    }
+  }
+  private String brokerUrl;
+  private String keySerializer = "org.apache.kafka.common.serialization.StringSerializer";
+  private String valueSerializer = "org.apache.kafka.common.serialization.StringSerializer";
+  private int requiredAcks = 1;
+  private String kafkaTopic = Constants.ENRICHMENT_TOPIC;
+  private KafkaProducer kafkaProducer;
+  private String configPrefix = null;
+  private String zkQuorum = null;
+  private Map<String, Object> producerConfigs = new HashMap<>();
+
+  public KafkaWriter() {}
+
+  public KafkaWriter(String brokerUrl) {
+    this.brokerUrl = brokerUrl;
+  }
+
+  public KafkaWriter withZkQuorum(String zkQuorum) {
+    this.zkQuorum = zkQuorum;
+    return this;
+  }
+  public KafkaWriter withKeySerializer(String keySerializer) {
+    this.keySerializer = keySerializer;
+    return this;
+  }
+
+  public KafkaWriter withValueSerializer(String valueSerializer) {
+    this.valueSerializer = valueSerializer;
+    return this;
+  }
+
+  public KafkaWriter withRequiredAcks(Integer requiredAcks) {
+    this.requiredAcks = requiredAcks;
+    return this;
+  }
+
+  public KafkaWriter withTopic(String topic) {
+    this.kafkaTopic= topic;
+    return this;
+  }
+  public KafkaWriter withConfigPrefix(String prefix) {
+    this.configPrefix = prefix;
+    return this;
+  }
+
+  public KafkaWriter withProducerConfigs(Map<String, Object> extraConfigs) {
+    this.producerConfigs = extraConfigs;
+    return this;
+  }
+
+  public Optional<String> getConfigPrefix() {
+    return Optional.ofNullable(configPrefix);
+  }
+
+  @Override
+  public void configure(String sensorName, WriterConfiguration configuration) {
+    Map<String, Object> configMap = configuration.getSensorConfig(sensorName);
+    String brokerUrl = Configurations.BROKER.getAndConvert(getConfigPrefix(), configMap, String.class);
+    if(brokerUrl != null) {
+      this.brokerUrl = brokerUrl;
+    }
+    String zkQuorum = Configurations.ZK_QUORUM.getAndConvert(getConfigPrefix(), configMap, String.class);
+    if(zkQuorum != null) {
+      withZkQuorum(zkQuorum);
+    }
+    String keySerializer = Configurations.KEY_SERIALIZER.getAndConvert(getConfigPrefix(), configMap, String.class);
+    if(keySerializer != null) {
+      withKeySerializer(keySerializer);
+    }
+    String valueSerializer = Configurations.VALUE_SERIALIZER.getAndConvert(getConfigPrefix(), configMap, String.class);
+    if(valueSerializer != null) {
+      withValueSerializer(keySerializer);
+    }
+    Integer requiredAcks = Configurations.REQUIRED_ACKS.getAndConvert(getConfigPrefix(), configMap, Integer.class);
+    if(requiredAcks!= null) {
+      withRequiredAcks(requiredAcks);
+    }
+    String topic = Configurations.TOPIC.getAndConvert(getConfigPrefix(), configMap, String.class);
+    if(topic != null) {
+      withTopic(topic);
+    }
+    Map<String, Object> producerConfigs = (Map)Configurations.PRODUCER_CONFIGS.get(getConfigPrefix(), configMap);
+    if(producerConfigs != null) {
+      withProducerConfigs(producerConfigs);
+    }
+  }
+
+  public Map<String, Object> createProducerConfigs() {
+    Map<String, Object> producerConfig = new HashMap<>();
+    producerConfig.put("bootstrap.servers", brokerUrl);
+    producerConfig.put("key.serializer", keySerializer);
+    producerConfig.put("value.serializer", valueSerializer);
+    producerConfig.put("request.required.acks", requiredAcks);
+    producerConfig.putAll(producerConfigs == null?new HashMap<>():producerConfigs);
+    return producerConfig;
+  }
+
+  @Override
+  public void init() {
+    if(this.zkQuorum != null && this.brokerUrl == null) {
+      try {
+        this.brokerUrl = Joiner.on(",").join(KafkaUtils.INSTANCE.getBrokersFromZookeeper(this.zkQuorum));
+      } catch (Exception e) {
+        throw new IllegalStateException("Cannot read kafka brokers from zookeeper and you didn't specify them, giving up!", e);
+      }
+    }
+    this.kafkaProducer = new KafkaProducer<>(createProducerConfigs());
+  }
+
+  @SuppressWarnings("unchecked")
+  @Override
+  public void write(String sourceType, WriterConfiguration configurations, Tuple tuple, JSONObject message) throws Exception {
+    kafkaProducer.send(new ProducerRecord<String, String>(kafkaTopic, message.toJSONString()));
+  }
+
+  @Override
+  public void close() throws Exception {
+    kafkaProducer.close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java
new file mode 100644
index 0000000..49bab6f
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.message;
+
+import backtype.storm.tuple.Tuple;
+import org.json.simple.JSONObject;
+
+public interface MessageGetter {
+  JSONObject getMessage(Tuple t);
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java
new file mode 100644
index 0000000..f25ea62
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.message;
+
+
+import backtype.storm.tuple.Tuple;
+import org.json.simple.JSONObject;
+
+public enum MessageGetters implements MessageGetter{
+   RAW(RawMessageGetter.DEFAULT)
+  ,NAMED(NamedMessageGetter.DEFAULT)
+  ;
+  MessageGetter getter;
+  MessageGetters(MessageGetter getter) {
+    this.getter = getter;
+  }
+  @Override
+  public JSONObject getMessage(Tuple t) {
+    return getter.getMessage(t);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java
new file mode 100644
index 0000000..70cfab2
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.message;
+
+import backtype.storm.tuple.Tuple;
+import org.json.simple.JSONObject;
+
+public class NamedMessageGetter implements MessageGetter {
+  public static NamedMessageGetter DEFAULT = new NamedMessageGetter("message");
+  private String messageName;
+  public NamedMessageGetter(String name) {
+    this.messageName = name;
+  }
+  @Override
+  public JSONObject getMessage(Tuple tuple) {
+    return (JSONObject)tuple.getValueByField(messageName);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java
new file mode 100644
index 0000000..16c0173
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.message;
+
+import backtype.storm.tuple.Tuple;
+import org.apache.metron.common.utils.JSONUtils;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+
+import java.io.UnsupportedEncodingException;
+
+public class RawMessageGetter implements MessageGetter {
+  public static RawMessageGetter DEFAULT = new RawMessageGetter(0);
+  private ThreadLocal<JSONParser> parser = new ThreadLocal<JSONParser>() {
+    @Override
+    protected JSONParser initialValue() {
+      return new JSONParser();
+    }
+  };
+  int position = 0;
+  public RawMessageGetter(int position) {
+    this.position = position;
+  }
+  @Override
+  public JSONObject getMessage(Tuple t) {
+    byte[] data = t.getBinary(position);
+    try {
+      return (JSONObject) parser.get().parse(new String(data, "UTF8"));
+    } catch (Exception e) {
+      throw new IllegalStateException(e.getMessage(), e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/NoopWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/NoopWriterTest.java b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/NoopWriterTest.java
new file mode 100644
index 0000000..3a5a762
--- /dev/null
+++ b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/NoopWriterTest.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class NoopWriterTest {
+  @Test
+  public void testFixedLatencyConfig() {
+    NoopWriter writer = new NoopWriter().withLatency("10");
+    Assert.assertTrue(writer.sleepFunction instanceof NoopWriter.FixedLatency);
+    NoopWriter.FixedLatency sleepFunction = (NoopWriter.FixedLatency)writer.sleepFunction;
+    Assert.assertEquals(10, sleepFunction.getLatency());
+  }
+
+  private void ensureRandomLatencyConfig(String latencyConfig, int min, int max) {
+    NoopWriter writer = new NoopWriter().withLatency(latencyConfig);
+    Assert.assertTrue(writer.sleepFunction instanceof NoopWriter.RandomLatency);
+    NoopWriter.RandomLatency sleepFunction = (NoopWriter.RandomLatency)writer.sleepFunction;
+    Assert.assertEquals(min, sleepFunction.getMin());
+    Assert.assertEquals(max, sleepFunction.getMax());
+  }
+
+  @Test
+  public void testRandomLatencyConfig() {
+    ensureRandomLatencyConfig("10,20", 10, 20);
+    ensureRandomLatencyConfig("10, 20", 10, 20);
+    ensureRandomLatencyConfig("10 ,20", 10, 20);
+    ensureRandomLatencyConfig("10 , 20", 10, 20);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/kafka/KafkaWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/kafka/KafkaWriterTest.java b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/kafka/KafkaWriterTest.java
new file mode 100644
index 0000000..1b95430
--- /dev/null
+++ b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/kafka/KafkaWriterTest.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.kafka;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class KafkaWriterTest {
+
+
+  public static final String SENSOR_TYPE = "test";
+  public WriterConfiguration createConfiguration(final Map<String, Object> parserConfig) {
+    ParserConfigurations configurations = new ParserConfigurations();
+    configurations.updateSensorParserConfig( SENSOR_TYPE
+                                           , new SensorParserConfig() {{
+                                              setParserConfig(parserConfig);
+                                                                      }}
+                                           );
+    return new ParserWriterConfiguration(configurations);
+  }
+
+  @Test
+  public void testHappyPathGlobalConfig() throws Exception {
+    KafkaWriter writer = new KafkaWriter();
+    WriterConfiguration configuration = createConfiguration(
+            new HashMap<String, Object>() {{
+              put("kafka.brokerUrl" , "localhost:6667");
+              put("kafka.topic" , SENSOR_TYPE);
+              put("kafka.producerConfigs" , ImmutableMap.of("key1", 1, "key2", "value2"));
+            }}
+    );
+
+    writer.configure(SENSOR_TYPE, configuration);
+    Map<String, Object> producerConfigs = writer.createProducerConfigs();
+    Assert.assertEquals(producerConfigs.get("bootstrap.servers"), "localhost:6667");
+    Assert.assertEquals(producerConfigs.get("key.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
+    Assert.assertEquals(producerConfigs.get("value.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
+    Assert.assertEquals(producerConfigs.get("request.required.acks"), 1);
+    Assert.assertEquals(producerConfigs.get("key1"), 1);
+    Assert.assertEquals(producerConfigs.get("key2"), "value2");
+  }
+
+  @Test
+  public void testHappyPathGlobalConfigWithPrefix() throws Exception {
+    KafkaWriter writer = new KafkaWriter();
+    writer.withConfigPrefix("prefix");
+    WriterConfiguration configuration = createConfiguration(
+            new HashMap<String, Object>() {{
+              put("prefix.kafka.brokerUrl" , "localhost:6667");
+              put("prefix.kafka.topic" , SENSOR_TYPE);
+              put("prefix.kafka.producerConfigs" , ImmutableMap.of("key1", 1, "key2", "value2"));
+            }}
+    );
+
+    writer.configure(SENSOR_TYPE, configuration);
+    Map<String, Object> producerConfigs = writer.createProducerConfigs();
+    Assert.assertEquals(producerConfigs.get("bootstrap.servers"), "localhost:6667");
+    Assert.assertEquals(producerConfigs.get("key.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
+    Assert.assertEquals(producerConfigs.get("value.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
+    Assert.assertEquals(producerConfigs.get("request.required.acks"), 1);
+    Assert.assertEquals(producerConfigs.get("key1"), 1);
+    Assert.assertEquals(producerConfigs.get("key2"), "value2");
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/pom.xml b/metron-platform/pom.xml
index ec274f0..1f8d5f1 100644
--- a/metron-platform/pom.xml
+++ b/metron-platform/pom.xml
@@ -51,6 +51,8 @@
 		<module>metron-integration-test</module>
 		<module>metron-test-utilities</module>
 		<module>metron-api</module>
+		<module>metron-indexing</module>
+		<module>metron-writer</module>
 		<module>metron-hbase</module>
 		<module>elasticsearch-shaded</module>
 		<module>metron-elasticsearch</module>


[5/5] incubator-metron git commit: METRON-154: Decouple enrichment and indexing closes apache/incubator-metron#192

Posted by ce...@apache.org.
METRON-154: Decouple enrichment and indexing closes apache/incubator-metron#192


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/5ffcef8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/5ffcef8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/5ffcef8d

Branch: refs/heads/master
Commit: 5ffcef8d4a9ec0788dca094f54b07d99c44c6bef
Parents: 1c568bf
Author: cstella <ce...@gmail.com>
Authored: Wed Jul 20 09:14:16 2016 -0400
Committer: cstella <ce...@gmail.com>
Committed: Wed Jul 20 09:14:16 2016 -0400

----------------------------------------------------------------------
 metron-deployment/amazon-ec2/conf/defaults.yml  |   1 +
 .../inventory/full-dev-platform/group_vars/all  |   3 +-
 .../inventory/metron_example/group_vars/all     |   1 +
 .../roles/metron_kafka_topics/defaults/main.yml |  12 +-
 .../roles/metron_streaming/defaults/main.yml    |   4 +
 .../metron_streaming/tasks/copy_bundles.yml     |  10 +
 .../roles/metron_streaming/tasks/topologies.yml |  28 +-
 .../roles/monit/tasks/monit-definitions.yml     |   8 +-
 metron-deployment/roles/monit/tasks/scripts.yml |   2 +
 .../monit/enrichment-elasticsearch.monit        |  26 -
 .../monit/templates/monit/enrichment-solr.monit |  26 -
 .../monit/templates/monit/enrichment.monit      |  26 +
 .../monit/indexing-elasticsearch.monit          |  26 +
 .../monit/templates/monit/indexing-solr.monit   |  26 +
 metron-platform/metron-common/pom.xml           |   4 +
 .../org/apache/metron/common/Constants.java     |   2 +
 .../apache/metron/common/utils/BloomFilter.java |  63 +++
 .../apache/metron/common/utils/KafkaUtils.java  |  54 ++
 .../metron/common/writer/AbstractWriter.java    |  26 -
 .../common/writer/BulkWriterComponent.java      | 130 -----
 .../apache/metron/common/writer/NoopWriter.java | 143 ------
 .../common/writer/WriterToBulkWriter.java       |  54 --
 .../metron/common/writer/NoopWriterTest.java    |  48 --
 metron-platform/metron-elasticsearch/pom.xml    |  29 ++
 .../src/main/config/elasticsearch.properties    |  55 +--
 .../scripts/start_elasticsearch_topology.sh     |   2 +-
 .../ElasticsearchEnrichmentIntegrationTest.java |  98 ----
 .../ElasticsearchIndexingIntegrationTest.java   | 100 ++++
 metron-platform/metron-enrichment/pom.xml       |  72 ++-
 .../src/main/assembly/assembly.xml              |   8 +
 .../src/main/config/enrichment.properties       |  82 ++++
 .../src/main/flux/enrichment/remote.yaml        |  69 +--
 .../src/main/flux/enrichment/test.yaml          |  64 +--
 .../enrichment/bolt/BulkMessageWriterBolt.java  |  83 ----
 .../accesstracker/BloomAccessTracker.java       |  32 +-
 .../writer/SimpleHbaseEnrichmentWriter.java     | 275 +++++++++++
 .../hbase/SimpleHbaseEnrichmentWriter.java      | 282 -----------
 .../apache/metron/writer/hdfs/HdfsWriter.java   |  96 ----
 .../writer/hdfs/SourceAwareMoveAction.java      |  48 --
 .../writer/hdfs/SourceFileNameFormat.java       |  48 --
 .../metron/writer/hdfs/SourceHandler.java       | 160 ------
 .../main/scripts/start_enrichment_topology.sh   |  22 +
 .../bolt/BulkMessageWriterBoltTest.java         |   3 +-
 .../integration/EnrichmentIntegrationTest.java  | 426 ++++++++++++++++
 .../components/ConfigUploadComponent.java       | 114 +++++
 .../integration/mock/MockGeoAdapter.java        |  64 +++
 .../integration/mock/MockHBaseConnector.java    |  52 ++
 .../integration/mock/MockTableProvider.java     |  45 ++
 .../integration/utils/SampleUtil.java           |  57 +++
 metron-platform/metron-indexing/pom.xml         | 253 ++++++++++
 .../src/main/assembly/assembly.xml              |  57 +++
 .../src/main/flux/indexing/remote.yaml          | 152 ++++++
 .../integration/IndexingIntegrationTest.java    | 240 +++++++++
 metron-platform/metron-integration-test/pom.xml |   5 -
 .../metron/integration/BaseIntegrationTest.java |   1 -
 .../integration/EnrichmentIntegrationTest.java  | 489 -------------------
 .../components/ConfigUploadComponent.java       | 114 -----
 .../metron/integration/mock/MockGeoAdapter.java |  64 ---
 .../integration/mock/MockHBaseConnector.java    |  52 --
 .../integration/mock/MockTableProvider.java     |  45 --
 .../metron/integration/utils/SampleUtil.java    |  57 ---
 metron-platform/metron-parsers/pom.xml          |  16 +-
 .../src/main/flux/asa/remote.yaml               |   2 +-
 .../metron-parsers/src/main/flux/asa/test.yaml  |   2 +-
 .../src/main/flux/fireeye/remote.yaml           |   2 +-
 .../src/main/flux/fireeye/test.yaml             |   2 +-
 .../src/main/flux/ise/remote.yaml               |   2 +-
 .../metron-parsers/src/main/flux/ise/test.yaml  |   2 +-
 .../src/main/flux/lancope/remote.yaml           |   2 +-
 .../src/main/flux/lancope/test.yaml             |   2 +-
 .../src/main/flux/paloalto/remote.yaml          |   2 +-
 .../src/main/flux/paloalto/test.yaml            |   2 +-
 .../src/main/flux/sourcefire/remote.yaml        |   2 +-
 .../src/main/flux/sourcefire/test.yaml          |   2 +-
 .../src/main/flux/websphere/remote.yaml         |   2 +-
 .../src/main/flux/websphere/test.yaml           |   2 +-
 .../apache/metron/parsers/bolt/ParserBolt.java  |  10 -
 .../metron/parsers/bolt/WriterHandler.java      |   4 +-
 .../parsers/topology/ParserTopologyBuilder.java |   4 +-
 .../metron/parsers/writer/KafkaWriter.java      | 164 -------
 .../integration/ParserIntegrationTest.java      |   6 +-
 .../metron/parsers/writer/KafkaWriterTest.java  |  95 ----
 .../SimpleHBaseEnrichmentWriterTest.java        |   5 +-
 ...pleHbaseEnrichmentWriterIntegrationTest.java |  14 +-
 .../integration/WriterBoltIntegrationTest.java  |  17 +-
 metron-platform/metron-solr/pom.xml             |  50 +-
 .../metron-solr/src/main/config/solr.properties |  61 +--
 .../src/main/scripts/start_solr_topology.sh     |   2 +-
 .../SolrEnrichmentIntegrationTest.java          | 114 -----
 .../SolrIndexingIntegrationTest.java            | 114 +++++
 .../metron/solr/writer/SolrWriterTest.java      |   4 +-
 metron-platform/metron-writer/pom.xml           | 231 +++++++++
 .../apache/metron/writer/AbstractWriter.java    |  26 +
 .../metron/writer/BulkWriterComponent.java      | 127 +++++
 .../org/apache/metron/writer/NoopWriter.java    | 141 ++++++
 .../metron/writer/WriterToBulkWriter.java       |  58 +++
 .../writer/bolt/BulkMessageWriterBolt.java      | 114 +++++
 .../apache/metron/writer/hdfs/HdfsWriter.java   |  96 ++++
 .../writer/hdfs/SourceAwareMoveAction.java      |  48 ++
 .../writer/hdfs/SourceFileNameFormat.java       |  48 ++
 .../metron/writer/hdfs/SourceHandler.java       | 160 ++++++
 .../apache/metron/writer/kafka/KafkaWriter.java | 181 +++++++
 .../metron/writer/message/MessageGetter.java    |  26 +
 .../metron/writer/message/MessageGetters.java   |  37 ++
 .../writer/message/NamedMessageGetter.java      |  34 ++
 .../metron/writer/message/RawMessageGetter.java |  50 ++
 .../apache/metron/writer/NoopWriterTest.java    |  48 ++
 .../metron/writer/kafka/KafkaWriterTest.java    |  88 ++++
 metron-platform/pom.xml                         |   2 +
 109 files changed, 4044 insertions(+), 2817 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/amazon-ec2/conf/defaults.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/conf/defaults.yml b/metron-deployment/amazon-ec2/conf/defaults.yml
index 2874b2c..9e05bbc 100644
--- a/metron-deployment/amazon-ec2/conf/defaults.yml
+++ b/metron-deployment/amazon-ec2/conf/defaults.yml
@@ -30,6 +30,7 @@ services_to_start:
   - bro-parser
   - snort-parser
   - enrichment
+  - indexing
 
 # the ami for cent6 by region
 amis_by_region:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/inventory/full-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all b/metron-deployment/inventory/full-dev-platform/group_vars/all
index 2e87a29..6afe4ae 100644
--- a/metron-deployment/inventory/full-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/full-dev-platform/group_vars/all
@@ -21,15 +21,14 @@ services_to_start:
   - elasticsearch
   - pcap-service
   - kibana
-  - yaf
   - snort
   - snort-logs
   - bro
   - pcap-replay
-  - yaf-parser
   - bro-parser
   - snort-parser
   - enrichment
+  - indexing
 
 # ambari
 ambari_host: "{{ groups.ambari_master[0] }}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/metron_example/group_vars/all b/metron-deployment/inventory/metron_example/group_vars/all
index bfa2469..a0d9257 100644
--- a/metron-deployment/inventory/metron_example/group_vars/all
+++ b/metron-deployment/inventory/metron_example/group_vars/all
@@ -30,6 +30,7 @@ services_to_start:
   - bro-parser
   - snort-parser
   - enrichment
+  - indexing
 
 #Ansible Variables
 ansible_ssh_private_key_file: /Path/to/private/key/file #Change This

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/metron_kafka_topics/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_kafka_topics/defaults/main.yml b/metron-deployment/roles/metron_kafka_topics/defaults/main.yml
index aac9a06..fa0ee02 100644
--- a/metron-deployment/roles/metron_kafka_topics/defaults/main.yml
+++ b/metron-deployment/roles/metron_kafka_topics/defaults/main.yml
@@ -17,10 +17,12 @@
 ---
 kafka_home: /usr/hdp/current/kafka-broker/
 topics_to_create:
-  - { topic: "pcap",            num_partitions: 1, replication_factor: 1, retention_gb: 10 }
-  - { topic: "bro",             num_partitions: 1, replication_factor: 1, retention_gb: 10 }
-  - { topic: "yaf",             num_partitions: 1, replication_factor: 1, retention_gb: 10 }
-  - { topic: "snort",           num_partitions: 1, replication_factor: 1, retention_gb: 10 }
-  - { topic: "enrichments",     num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+  - { topic: "pcap",        num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+  - { topic: "bro",         num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+  - { topic: "yaf",         num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+  - { topic: "snort",       num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+  - { topic: "enrichments", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
   - { topic: "parser_invalid",  num_partitions: 1, replication_factor: 1, retention_gb: 10 }
   - { topic: "parser_error",    num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+  - { topic: "indexing", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+  - { topic: "indexing_error", num_partitions: 1, replication_factor: 1, retention_gb: 10 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/defaults/main.yml b/metron-deployment/roles/metron_streaming/defaults/main.yml
index cbec601..8618c38 100644
--- a/metron-deployment/roles/metron_streaming/defaults/main.yml
+++ b/metron-deployment/roles/metron_streaming/defaults/main.yml
@@ -20,11 +20,13 @@ metron_solr_jar_name: metron-solr-{{ metron_version }}.jar
 metron_elasticsearch_jar_name: metron-elasticsearch-{{ metron_version }}.jar
 metron_pcap_jar_name: metron-pcap-backend-{{ metron_version }}.jar
 metron_parsers_jar_name: metron-parsers-{{ metron_version }}.jar
+metron_enrichment_jar_name: metron-enrichment-{{ metron_version }}-uber.jar
 
 metron_common_bundle_name: metron-common-{{ metron_version }}-archive.tar.gz
 metron_pcap_bundle_name: metron-pcap-backend-{{ metron_version }}-archive.tar.gz
 metron_data_management_bundle_name: metron-data-management-{{ metron_version }}-archive.tar.gz
 metron_enrichment_bundle_name: metron-enrichment-{{ metron_version }}-archive.tar.gz
+metron_indexing_bundle_name: metron-indexing-{{ metron_version }}-archive.tar.gz
 metron_solr_bundle_name: metron-solr-{{ metron_version }}-archive.tar.gz
 metron_elasticsearch_bundle_name: metron-elasticsearch-{{ metron_version }}-archive.tar.gz
 metron_parsers_bundle_name: metron-parsers-{{ metron_version }}-archive.tar.gz
@@ -32,6 +34,7 @@ metron_pcap_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-pcap-b
 metron_common_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-common/target/{{ metron_common_bundle_name }}"
 metron_data_management_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-data-management/target/{{ metron_data_management_bundle_name }}"
 metron_enrichment_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-enrichment/target/{{ metron_enrichment_bundle_name }}"
+metron_indexing_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-indexing/target/{{ metron_indexing_bundle_name }}"
 metron_solr_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-solr/target/{{ metron_solr_bundle_name }}"
 metron_elasticsearch_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-elasticsearch/target/{{ metron_elasticsearch_bundle_name }}"
 metron_parsers_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-parsers/target/{{ metron_parsers_bundle_name }}"
@@ -42,6 +45,7 @@ zookeeper_global_config_path: "{{ zookeeper_config_path }}/global.json"
 metron_pcap_properties_config_path: "{{ metron_directory }}/config/pcap.properties"
 metron_solr_properties_config_path: "{{ metron_directory }}/config/solr.properties"
 metron_elasticsearch_properties_config_path: "{{ metron_directory }}/config/elasticsearch.properties"
+metron_enrichment_properties_config_path: "{{ metron_directory }}/config/enrichment.properties"
 metron_parsers_properties_config_path: "{{ metron_directory }}/config/parsers.properties"
 hbase_config_path: "/etc/hbase/conf"
 hdfs_config_path: "/etc/hadoop/conf"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml b/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml
index 97fb259..519b7d3 100644
--- a/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml
+++ b/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml
@@ -30,6 +30,12 @@
     src: "{{ metron_enrichment_bundle_path }}"
     dest: "{{ metron_directory }}"
 
+- name: Copy Metron Indexing bundle
+  copy:
+    src: "{{ metron_indexing_bundle_path }}"
+    dest: "{{ metron_directory }}"
+
+
 - name: Copy Metron Parsers bundle
   copy:
     src: "{{ metron_parsers_bundle_path }}"
@@ -58,6 +64,7 @@
     - tar xzvf metron-solr*.tar.gz
     - tar xzvf metron-elasticsearch*.tar.gz
     - tar xzvf metron-enrichment*.tar.gz
+    - tar xzvf metron-indexing*.tar.gz
     - tar xzvf metron-parsers*.tar.gz
     - tar xzvf metron-data-management*.tar.gz
     - tar xzvf metron-common*.tar.gz
@@ -67,6 +74,9 @@
 - name: Add *-site.xml files to topology jars
   shell: "cd {{ item.config_path }} && jar -uf {{ metron_directory }}/lib/{{ item.jar_name }} {{ item.file_name }}"
   with_items:
+      - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_enrichment_jar_name }}", file_name: "hbase-site.xml" }
+      - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_enrichment_jar_name }}", file_name: "core-site.xml" }
+      - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_enrichment_jar_name }}", file_name: "hdfs-site.xml" }
       - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hbase-site.xml" }
       - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "core-site.xml" }
       - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hdfs-site.xml" }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/metron_streaming/tasks/topologies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/topologies.yml b/metron-deployment/roles/metron_streaming/tasks/topologies.yml
index 2010eae..0ca0b3d 100644
--- a/metron-deployment/roles/metron_streaming/tasks/topologies.yml
+++ b/metron-deployment/roles/metron_streaming/tasks/topologies.yml
@@ -29,18 +29,14 @@
     - { regexp: "kafka.pcap.out=", line: "kafka.pcap.out={{ pcap_hdfs_path }}" }
     - { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" }
 
-- name: Configure Metron Solr topology
+- name: Configure Metron Enrichment topology
   lineinfile: >
-    dest={{ metron_solr_properties_config_path }}
+    dest={{ metron_enrichment_properties_config_path }}
     regexp="{{ item.regexp }}"
     line="{{ item.line }}"
   with_items:
     - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
     - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
-    - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
-    - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
-    - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
-    - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
     - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
     - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" }
     - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
@@ -50,6 +46,17 @@
     - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" }
     - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
     - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
+
+
+- name: Configure Metron Solr topology
+  lineinfile: >
+    dest={{ metron_solr_properties_config_path }}
+    regexp="{{ item.regexp }}"
+    line="{{ item.line }}"
+  with_items:
+    - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
+    - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
+    - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
     - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
     - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
     - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
@@ -67,15 +74,6 @@
     - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
     - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
     - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
-    - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
-    - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" }
-    - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
-    - { regexp: "threat.intel.simple.hbase.table=", line: "threat.intel.simple.hbase.table={{ threatintel_hbase_table }}" }
-    - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" }
-    - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" }
-    - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" }
-    - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
-    - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
     - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
     - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
     - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/monit/tasks/monit-definitions.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/monit-definitions.yml b/metron-deployment/roles/monit/tasks/monit-definitions.yml
index 3778e14..f9e3897 100644
--- a/metron-deployment/roles/monit/tasks/monit-definitions.yml
+++ b/metron-deployment/roles/monit/tasks/monit-definitions.yml
@@ -24,10 +24,14 @@
   template: src=monit/elasticsearch.monit dest={{ monit_config_home }}/elasticsearch.monit
   when: ("search" in group_names) and (install_elasticsearch | default(True))
 
-- name: Create monit definition for enrichment with elasticsearch
-  template: src=monit/enrichment-elasticsearch.monit dest={{ monit_config_home }}/enrichment-elasticsearch.monit
+- name: Create monit definition for indexing with elasticsearch
+  template: src=monit/indexing-elasticsearch.monit dest={{ monit_config_home }}/indexing-elasticsearch.monit
   when: ("enrichment" in group_names) and (install_elasticsearch | default(True))
 
+- name: Create monit definition for enrichment
+  template: src=monit/enrichment.monit dest={{ monit_config_home }}/enrichment.monit
+  when: ("enrichment" in group_names)
+
 - name: Create monit definition for kibana
   template: src=monit/kibana.monit dest={{ monit_config_home }}/kibana.monit
   when: ("web" in group_names) and (install_elasticsearch | default(True))

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/monit/tasks/scripts.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/scripts.yml b/metron-deployment/roles/monit/tasks/scripts.yml
index df1a443..43901d2 100644
--- a/metron-deployment/roles/monit/tasks/scripts.yml
+++ b/metron-deployment/roles/monit/tasks/scripts.yml
@@ -39,6 +39,7 @@
     - bro
     - pcap
     - enrichment
+    - indexing
 
 - name: Deploy topology status scripts
   template:
@@ -51,3 +52,4 @@
     - bro
     - pcap
     - enrichment
+    - indexing

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/monit/templates/monit/enrichment-elasticsearch.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/enrichment-elasticsearch.monit b/metron-deployment/roles/monit/templates/monit/enrichment-elasticsearch.monit
deleted file mode 100644
index ceae475..0000000
--- a/metron-deployment/roles/monit/templates/monit/enrichment-elasticsearch.monit
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-check program enrichment with path "{{ monit_home }}/status_enrichment_topology.sh"
-  start program "{{ metron_directory }}/bin/start_elasticsearch_topology.sh"
-  stop program "{{ monit_home }}/stop_enrichment_topology.sh"
-  if status != 0 then restart
-  group yaf
-  group bro
-  group snort
-  group enrichments
-  group metron

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/monit/templates/monit/enrichment-solr.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/enrichment-solr.monit b/metron-deployment/roles/monit/templates/monit/enrichment-solr.monit
deleted file mode 100644
index 94f4858..0000000
--- a/metron-deployment/roles/monit/templates/monit/enrichment-solr.monit
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-check program enrichment with path "{{ monit_home }}/status_enrichment_topology.sh"
-  start program "{{ metron_directory }}/bin/start_solr_topology.sh"
-  stop program "{{ monit_home }}/stop_enrichment_topology.sh"
-  if status != 0 then restart
-  group yaf
-  group bro
-  group snort
-  group enrichments
-  group metron

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/monit/templates/monit/enrichment.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/enrichment.monit b/metron-deployment/roles/monit/templates/monit/enrichment.monit
new file mode 100644
index 0000000..43d197e
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/enrichment.monit
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check program enrichment with path "{{ monit_home }}/status_enrichment_topology.sh"
+  start program "{{ metron_directory }}/bin/start_enrichment_topology.sh"
+  stop program "{{ monit_home }}/stop_enrichment_topology.sh"
+  if status != 0 then restart
+  group yaf
+  group bro
+  group snort
+  group enrichments
+  group metron

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/monit/templates/monit/indexing-elasticsearch.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/indexing-elasticsearch.monit b/metron-deployment/roles/monit/templates/monit/indexing-elasticsearch.monit
new file mode 100644
index 0000000..86992e4
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/indexing-elasticsearch.monit
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check program indexing with path "{{ monit_home }}/status_indexing_topology.sh"
+  start program "{{ metron_directory }}/bin/start_elasticsearch_topology.sh"
+  stop program "{{ monit_home }}/stop_indexing_topology.sh"
+  if status != 0 then restart
+  group yaf
+  group bro
+  group snort
+  group enrichments
+  group metron

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/monit/templates/monit/indexing-solr.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/indexing-solr.monit b/metron-deployment/roles/monit/templates/monit/indexing-solr.monit
new file mode 100644
index 0000000..432a0e6
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/indexing-solr.monit
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check program indexing with path "{{ monit_home }}/status_indexing_topology.sh"
+  start program "{{ metron_directory }}/bin/start_solr_topology.sh"
+  stop program "{{ monit_home }}/stop_indexing_topology.sh"
+  if status != 0 then restart
+  group yaf
+  group bro
+  group snort
+  group enrichments
+  group metron

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index 30162dd..d625402 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -294,6 +294,10 @@
                                     <pattern>com.google.common</pattern>
                                     <shadedPattern>org.apache.metron.guava</shadedPattern>
                                 </relocation>
+                                <relocation>
+                                    <pattern>org.apache.commons.beanutils</pattern>
+                                    <shadedPattern>org.apache.metron.beanutils</shadedPattern>
+                                </relocation>
                             </relocations>
                             <transformers>
                                 <transformer

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
index 1175e8b..d1f5748 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
@@ -27,6 +27,8 @@ public class Constants {
   public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000;
   public static final String SENSOR_TYPE = "source.type";
   public static final String ENRICHMENT_TOPIC = "enrichments";
+  public static final String INDEXING_TOPIC = "indexing";
+  public static final String INDEXING_ERROR_TOPIC = "indexing_error";
   public static final String DEFAULT_PARSER_ERROR_TOPIC = "parser_error";
   public static final String DEFAULT_PARSER_INVALID_TOPIC = "parser_invalid";
   public static final String ERROR_STREAM = "error";

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/BloomFilter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/BloomFilter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/BloomFilter.java
new file mode 100644
index 0000000..82172f5
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/BloomFilter.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.utils;
+
+import com.google.common.hash.Funnel;
+import com.google.common.hash.PrimitiveSink;
+
+import java.io.Serializable;
+import java.util.function.Function;
+
+public class BloomFilter<T> implements Serializable {
+
+  private static class BloomFunnel<T> implements Funnel<T>, Serializable {
+    Function<T, byte[]> serializer;
+    public BloomFunnel(Function<T, byte[]> serializer) {
+      this.serializer = serializer;
+    }
+    @Override
+    public void funnel(T obj, PrimitiveSink primitiveSink) {
+      primitiveSink.putBytes(serializer.apply(obj));
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      return this.getClass().equals(obj.getClass());
+    }
+
+    @Override
+    public int hashCode() {
+      return super.hashCode() * 31;
+    }
+  }
+  private com.google.common.hash.BloomFilter<T> filter;
+  public BloomFilter(Function<T, byte[]> serializer, int expectedInsertions, double falsePositiveRate) {
+    filter = com.google.common.hash.BloomFilter.create(new BloomFunnel<T>(serializer), expectedInsertions, falsePositiveRate);
+  }
+
+  public boolean mightContain(T key) {
+    return filter.mightContain(key);
+  }
+  public void add(T key) {
+    filter.put(key);
+  }
+  public void merge(BloomFilter<T> filter2) {
+    filter.putAll(filter2.filter);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
new file mode 100644
index 0000000..6d2af72
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.utils;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public enum KafkaUtils {
+  INSTANCE;
+  public List<String> getBrokersFromZookeeper(String zkQuorum) throws Exception {
+    RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+    CuratorFramework framework = CuratorFrameworkFactory.newClient(zkQuorum, retryPolicy);
+    framework.start();
+    try {
+      return getBrokersFromZookeeper(framework);
+    } finally {
+      framework.close();
+    }
+  }
+  public List<String> getBrokersFromZookeeper(CuratorFramework client) throws Exception {
+    List<String> ret = new ArrayList<>();
+    for(String id : client.getChildren().forPath("/brokers/ids")) {
+      byte[] data = client.getData().forPath("/brokers/ids/" + id);
+      String brokerInfoStr = new String(data);
+      Map<String, Object> brokerInfo = JSONUtils.INSTANCE.load(brokerInfoStr, new TypeReference<Map<String, Object>>() {
+      });
+      ret.add(brokerInfo.get("host") + ":" + brokerInfo.get("port"));
+    }
+    return ret;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/AbstractWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/AbstractWriter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/AbstractWriter.java
deleted file mode 100644
index 56a4e48..0000000
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/AbstractWriter.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.common.writer;
-
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-
-public abstract class AbstractWriter {
-  public AbstractWriter() {}
-  public abstract void configure(String sensorName, WriterConfiguration configuration);
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterComponent.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterComponent.java
deleted file mode 100644
index c0e4f37..0000000
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterComponent.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.common.writer;
-
-import backtype.storm.task.OutputCollector;
-import backtype.storm.tuple.Tuple;
-import com.google.common.collect.Iterables;
-import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.interfaces.BulkMessageWriter;
-import org.apache.metron.common.utils.ErrorUtils;
-import org.apache.metron.common.utils.MessageUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-import java.util.function.Function;
-
-public class BulkWriterComponent<MESSAGE_T> {
-  public static final Logger LOG = LoggerFactory
-            .getLogger(BulkWriterComponent.class);
-  private Map<String, Collection<Tuple>> sensorTupleMap = new HashMap<>();
-  private Map<String, List<MESSAGE_T>> sensorMessageMap = new HashMap<>();
-  private OutputCollector collector;
-  private boolean handleCommit = true;
-  private boolean handleError = true;
-  public BulkWriterComponent(OutputCollector collector) {
-    this.collector = collector;
-  }
-
-  public BulkWriterComponent(OutputCollector collector, boolean handleCommit, boolean handleError) {
-    this(collector);
-    this.handleCommit = handleCommit;
-    this.handleError = handleError;
-  }
-
-  public void commit(Iterable<Tuple> tuples) {
-    tuples.forEach(t -> collector.ack(t));
-    if(LOG.isDebugEnabled()) {
-      LOG.debug("Acking " + Iterables.size(tuples) + " tuples");
-    }
-  }
-
-  public void error(Throwable e, Iterable<Tuple> tuples) {
-    tuples.forEach(t -> collector.ack(t));
-    if(!Iterables.isEmpty(tuples)) {
-      LOG.error("Failing " + Iterables.size(tuples) + " tuples", e);
-      ErrorUtils.handleError(collector, e, Constants.ERROR_STREAM);
-    }
-  }
-
-  protected Collection<Tuple> createTupleCollection() {
-    return new ArrayList<>();
-  }
-
-
-  public void errorAll(Throwable e) {
-    for(Map.Entry<String, Collection<Tuple>> kv : sensorTupleMap.entrySet()) {
-      error(e, kv.getValue());
-      sensorTupleMap.remove(kv.getKey());
-      sensorMessageMap.remove(kv.getKey());
-    }
-  }
-
-  public void errorAll(String sensorType, Throwable e) {
-    error(e, Optional.ofNullable(sensorTupleMap.get(sensorType)).orElse(new ArrayList<>()));
-    sensorTupleMap.remove(sensorType);
-    sensorMessageMap.remove(sensorType);
-  }
-  public void write( String sensorType
-                   , Tuple tuple
-                   , MESSAGE_T message
-                   , BulkMessageWriter<MESSAGE_T> bulkMessageWriter
-                   , WriterConfiguration configurations
-                   ) throws Exception
-  {
-    int batchSize = configurations.getBatchSize(sensorType);
-    Collection<Tuple> tupleList = sensorTupleMap.get(sensorType);
-    if (tupleList == null) {
-      tupleList = createTupleCollection();
-    }
-    tupleList.add(tuple);
-    List<MESSAGE_T> messageList = sensorMessageMap.get(sensorType);
-    if (messageList == null) {
-      messageList = new ArrayList<>();
-    }
-    messageList.add(message);
-
-    if (tupleList.size() < batchSize) {
-      sensorTupleMap.put(sensorType, tupleList);
-      sensorMessageMap.put(sensorType, messageList);
-    } else {
-      try {
-        bulkMessageWriter.write(sensorType, configurations, tupleList, messageList);
-        if(handleCommit) {
-          commit(tupleList);
-        }
-
-      } catch (Throwable e) {
-        if(handleError) {
-          error(e, tupleList);
-        }
-        else {
-          throw e;
-        }
-      }
-      finally {
-        sensorTupleMap.remove(sensorType);
-        sensorMessageMap.remove(sensorType);
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/NoopWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/NoopWriter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/NoopWriter.java
deleted file mode 100644
index 3b27f75..0000000
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/NoopWriter.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.common.writer;
-
-import backtype.storm.tuple.Tuple;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Iterables;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.interfaces.BulkMessageWriter;
-import org.apache.metron.common.utils.ConversionUtils;
-
-import java.io.Closeable;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.function.Function;
-
-public class NoopWriter extends AbstractWriter implements BulkMessageWriter<Object> {
-
-  public static class RandomLatency implements Function<Void, Void> {
-    private int min;
-    private int max;
-
-    public RandomLatency(int min, int max) {
-      this.min = min;
-      this.max = max;
-    }
-
-    public int getMin() {
-      return min;
-    }
-    public int getMax() {
-      return max;
-    }
-
-    @Override
-    public Void apply(Void aVoid) {
-      int sleepMs = ThreadLocalRandom.current().nextInt(min, max + 1);
-      try {
-        Thread.sleep(sleepMs);
-      } catch (InterruptedException e) {
-      }
-      return null;
-    }
-  }
-
-  public static class FixedLatency implements Function<Void, Void> {
-    private int latency;
-    public FixedLatency(int latency) {
-      this.latency = latency;
-    }
-    public int getLatency() {
-      return latency;
-    }
-
-    @Override
-    public Void apply(Void aVoid) {
-      if(latency > 0) {
-        try {
-          Thread.sleep(latency);
-        } catch (InterruptedException e) {
-        }
-      }
-      return null;
-    }
-  }
-  Function<Void, Void> sleepFunction = null;
-
-  public NoopWriter withLatency(String sleepConfig) {
-    sleepFunction = getSleepFunction(sleepConfig);
-    return this;
-  }
-
-
-  private Function<Void, Void> getSleepFunction(String sleepConfig) {
-    String usageMessage = "Unexpected: " + sleepConfig + " Expected value: integer for a fixed sleep duration in milliseconds (e.g. 10) " +
-            "or a range of latencies separated by a comma (e.g. \"10, 20\") to sleep a random amount in that range.";
-    try {
-      if (sleepConfig.contains(",")) {
-        // random latency within a range.
-        Iterable<String> it = Splitter.on(',').split(sleepConfig);
-        Integer min = ConversionUtils.convert(Iterables.getFirst(it, "").trim(), Integer.class);
-        Integer max= ConversionUtils.convert(Iterables.getLast(it, "").trim(), Integer.class);
-        if (min != null && max != null) {
-          return new RandomLatency(min, max);
-        }
-      } else {
-        //fixed latency
-        Integer latency = ConversionUtils.convert(sleepConfig.trim(), Integer.class);
-        if(latency != null) {
-          return new FixedLatency(latency);
-        }
-      }
-    }
-    catch(Throwable t) {
-      throw new IllegalArgumentException(usageMessage, t);
-    }
-    throw new IllegalArgumentException(usageMessage);
-  }
-
-  @Override
-  public void configure(String sensorName, WriterConfiguration configuration) {
-    Map<String, Object> config = configuration.getSensorConfig(sensorName);
-    if(config != null) {
-      Object noopLatency = config.get("noopLatency");
-      if(noopLatency != null) {
-        sleepFunction = getSleepFunction(noopLatency.toString());
-      }
-    }
-  }
-
-  @Override
-  public void init(Map stormConf, WriterConfiguration config) throws Exception {
-  }
-
-  @Override
-  public void write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<Object> messages) throws Exception {
-    if(sleepFunction != null) {
-      sleepFunction.apply(null);
-    }
-  }
-
-  @Override
-  public void close() throws Exception {
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/WriterToBulkWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/WriterToBulkWriter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/WriterToBulkWriter.java
deleted file mode 100644
index b0bde6c..0000000
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/WriterToBulkWriter.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.common.writer;
-
-import backtype.storm.tuple.Tuple;
-import com.google.common.collect.Iterables;
-import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFacade;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.interfaces.BulkMessageWriter;
-import org.apache.metron.common.interfaces.MessageWriter;
-
-import java.util.List;
-import java.util.Map;
-
-public class WriterToBulkWriter<MESSAGE_T> implements BulkMessageWriter<MESSAGE_T> {
-  MessageWriter<MESSAGE_T> messageWriter;
-
-  public WriterToBulkWriter(MessageWriter<MESSAGE_T> messageWriter) {
-    this.messageWriter = messageWriter;
-  }
-  @Override
-  public void init(Map stormConf, WriterConfiguration config) throws Exception {
-    messageWriter.init();
-  }
-
-  @Override
-  public void write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<MESSAGE_T> messages) throws Exception {
-    if(messages.size() > 1) {
-      throw new IllegalStateException("WriterToBulkWriter expects a batch of exactly 1");
-    }
-    messageWriter.write(sensorType, configurations, Iterables.getFirst(tuples, null), Iterables.getFirst(messages, null));
-  }
-
-  @Override
-  public void close() throws Exception {
-    messageWriter.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/test/java/org/apache/metron/common/writer/NoopWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/writer/NoopWriterTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/writer/NoopWriterTest.java
deleted file mode 100644
index e40ce7c..0000000
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/writer/NoopWriterTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.common.writer;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-public class NoopWriterTest {
-  @Test
-  public void testFixedLatencyConfig() {
-    NoopWriter writer = new NoopWriter().withLatency("10");
-    Assert.assertTrue(writer.sleepFunction instanceof NoopWriter.FixedLatency);
-    NoopWriter.FixedLatency sleepFunction = (NoopWriter.FixedLatency)writer.sleepFunction;
-    Assert.assertEquals(10, sleepFunction.getLatency());
-  }
-
-  private void ensureRandomLatencyConfig(String latencyConfig, int min, int max) {
-    NoopWriter writer = new NoopWriter().withLatency(latencyConfig);
-    Assert.assertTrue(writer.sleepFunction instanceof NoopWriter.RandomLatency);
-    NoopWriter.RandomLatency sleepFunction = (NoopWriter.RandomLatency)writer.sleepFunction;
-    Assert.assertEquals(min, sleepFunction.getMin());
-    Assert.assertEquals(max, sleepFunction.getMax());
-  }
-
-  @Test
-  public void testRandomLatencyConfig() {
-    ensureRandomLatencyConfig("10,20", 10, 20);
-    ensureRandomLatencyConfig("10, 20", 10, 20);
-    ensureRandomLatencyConfig("10 ,20", 10, 20);
-    ensureRandomLatencyConfig("10 , 20", 10, 20);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index 4db3c07..3ecd84d 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -40,6 +40,16 @@
             <groupId>org.apache.metron</groupId>
             <artifactId>metron-enrichment</artifactId>
             <version>${project.parent.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-databind</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-annotations</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.storm</groupId>
@@ -132,6 +142,25 @@
             <version>${global_mockito_version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-indexing</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-indexing</artifactId>
+            <version>${project.parent.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-enrichment</artifactId>
+            <version>${project.parent.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
     <reporting>
         <plugins>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
index e2370ab..a1516d6 100644
--- a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
+++ b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
@@ -14,22 +14,19 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+##### Storm #####
+indexing.workers=1
+indexing.executors=0
 
 ##### Kafka #####
 
 kafka.zk=node1:2181
 kafka.broker=node1:6667
-spout.kafka.topic.asa=asa
-spout.kafka.topic.bro=bro
-spout.kafka.topic.fireeye=fireeye
-spout.kafka.topic.ise=ise
-spout.kafka.topic.lancope=lancope
-spout.kafka.topic.paloalto=paloalto
-spout.kafka.topic.pcap=pcap
-spout.kafka.topic.snort=snort
-spout.kafka.topic.yaf=yaf
+kafka.start=WHERE_I_LEFT_OFF
 
 ##### Indexing #####
+index.input.topic=indexing
+index.error.topic=indexing_error
 writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter
 
 ##### ElasticSearch #####
@@ -38,13 +35,6 @@ es.ip=10.22.0.214
 es.port=9300
 es.clustername=elasticsearch
 
-##### MySQL #####
-
-mysql.ip=10.22.0.214
-mysql.port=3306
-mysql.username=root
-mysql.password=hadoop123
-
 ##### Metrics #####
 
 #reporters
@@ -63,23 +53,6 @@ org.apache.metron.metrics.TelemetryParserBolt.emits=true
 org.apache.metron.metrics.TelemetryParserBolt.fails=true
 
 
-#GenericEnrichmentBolt
-org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
-org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
-org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
-
-
-#TelemetryIndexingBolt
-org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
-org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
-org.apache.metron.metrics.TelemetryIndexingBolt.fails=true
-
-##### Host Enrichment #####
-
-org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
-{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
-{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
-
 ##### HDFS #####
 
 bolt.hdfs.batch.size=5000
@@ -91,19 +64,3 @@ bolt.hdfs.finished.file.path=/paloalto/rotated
 bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
 index.hdfs.output=/tmp/metron/enriched
 
-##### HBase #####
-bolt.hbase.table.name=pcap
-bolt.hbase.table.fields=t:value
-bolt.hbase.table.key.tuple.field.name=key
-bolt.hbase.table.timestamp.tuple.field.name=timestamp
-bolt.hbase.enable.batching=false
-bolt.hbase.write.buffer.size.in.bytes=2000000
-bolt.hbase.durability=SKIP_WAL
-bolt.hbase.partitioner.region.info.refresh.interval.mins=60
-
-##### Threat Intel #####
-
-threat.intel.tracker.table=
-threat.intel.tracker.cf=
-threat.intel.ip.table=
-threat.intel.ip.cf=

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh b/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh
index 9149d09..16f83d3 100755
--- a/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh
+++ b/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh
@@ -19,4 +19,4 @@
 METRON_VERSION=${project.version}
 METRON_HOME=/usr/metron/$METRON_VERSION
 TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION.jar
-storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/enrichment/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties
+storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java
deleted file mode 100644
index e0bbb89..0000000
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.elasticsearch.integration;
-
-import org.apache.metron.TestConstants;
-import org.apache.metron.common.interfaces.FieldNameConverter;
-import org.apache.metron.elasticsearch.writer.ElasticsearchFieldNameConverter;
-import org.apache.metron.integration.EnrichmentIntegrationTest;
-import org.apache.metron.integration.ComponentRunner;
-import org.apache.metron.integration.InMemoryComponent;
-import org.apache.metron.integration.Processor;
-import org.apache.metron.integration.ReadinessState;
-import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-public class ElasticsearchEnrichmentIntegrationTest extends EnrichmentIntegrationTest {
-
-  private String indexDir = "target/elasticsearch";
-  private String dateFormat = "yyyy.MM.dd.HH";
-  private String index = "yaf_index_" + new SimpleDateFormat(dateFormat).format(new Date());
-  private FieldNameConverter fieldNameConverter = new ElasticsearchFieldNameConverter();
-
-  @Override
-  public FieldNameConverter getFieldNameConverter() {
-    return fieldNameConverter;
-  }
-
-  @Override
-  public InMemoryComponent getSearchComponent(final Properties topologyProperties) {
-    return new ElasticSearchComponent.Builder()
-            .withHttpPort(9211)
-            .withIndexDir(new File(indexDir))
-            .build();
-  }
-
-  @Override
-  public Processor<List<Map<String, Object>>> getProcessor(final List<byte[]> inputMessages) {
-    return new Processor<List<Map<String, Object>>>() {
-      List<Map<String, Object>> docs = null;
-      public ReadinessState process(ComponentRunner runner) {
-        ElasticSearchComponent elasticSearchComponent = runner.getComponent("search", ElasticSearchComponent.class);
-        if (elasticSearchComponent.hasIndex(index)) {
-          List<Map<String, Object>> docsFromDisk;
-          try {
-            docs = elasticSearchComponent.getAllIndexedDocs(index, testSensorType + "_doc");
-            docsFromDisk = readDocsFromDisk(hdfsDir);
-            System.out.println(docs.size() + " vs " + inputMessages.size() + " vs " + docsFromDisk.size());
-          } catch (IOException e) {
-            throw new IllegalStateException("Unable to retrieve indexed documents.", e);
-          }
-          if (docs.size() < inputMessages.size() || docs.size() != docsFromDisk.size()) {
-            return ReadinessState.NOT_READY;
-          } else {
-            return ReadinessState.READY;
-          }
-        } else {
-          return ReadinessState.NOT_READY;
-        }
-      }
-
-      public List<Map<String, Object>> getResult() {
-        return docs;
-      }
-    };
-  }
-
-  @Override
-  public void setAdditionalProperties(Properties topologyProperties) {
-    topologyProperties.setProperty("writer.class.name", "org.apache.metron.elasticsearch.writer.ElasticsearchWriter");
-  }
-
-  @Override
-  public String cleanField(String field) {
-    return field;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java
new file mode 100644
index 0000000..b27c545
--- /dev/null
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.elasticsearch.integration;
+
+import org.apache.metron.common.interfaces.FieldNameConverter;
+import org.apache.metron.elasticsearch.writer.ElasticsearchFieldNameConverter;
+import org.apache.metron.indexing.integration.IndexingIntegrationTest;
+import org.apache.metron.integration.ComponentRunner;
+import org.apache.metron.integration.InMemoryComponent;
+import org.apache.metron.integration.Processor;
+import org.apache.metron.integration.ReadinessState;
+import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class ElasticsearchIndexingIntegrationTest extends IndexingIntegrationTest {
+
+  private String indexDir = "target/elasticsearch";
+  private String dateFormat = "yyyy.MM.dd.HH";
+  private String index = "yaf_index_" + new SimpleDateFormat(dateFormat).format(new Date());
+  private FieldNameConverter fieldNameConverter = new ElasticsearchFieldNameConverter();
+
+  @Override
+  public FieldNameConverter getFieldNameConverter() {
+    return fieldNameConverter;
+  }
+
+  @Override
+  public InMemoryComponent getSearchComponent(final Properties topologyProperties) {
+    return new ElasticSearchComponent.Builder()
+            .withHttpPort(9211)
+            .withIndexDir(new File(indexDir))
+            .build();
+  }
+
+  @Override
+  public Processor<List<Map<String, Object>>> getProcessor(final List<byte[]> inputMessages) {
+    return new Processor<List<Map<String, Object>>>() {
+      List<Map<String, Object>> docs = null;
+      public ReadinessState process(ComponentRunner runner) {
+        ElasticSearchComponent elasticSearchComponent = runner.getComponent("search", ElasticSearchComponent.class);
+        if (elasticSearchComponent.hasIndex(index)) {
+          List<Map<String, Object>> docsFromDisk;
+          try {
+            docs = elasticSearchComponent.getAllIndexedDocs(index, testSensorType + "_doc");
+            docsFromDisk = readDocsFromDisk(hdfsDir);
+            System.out.println(docs.size() + " vs " + inputMessages.size() + " vs " + docsFromDisk.size());
+          } catch (IOException e) {
+            throw new IllegalStateException("Unable to retrieve indexed documents.", e);
+          }
+          if (docs.size() < inputMessages.size() || docs.size() != docsFromDisk.size()) {
+            return ReadinessState.NOT_READY;
+          } else {
+            return ReadinessState.READY;
+          }
+        } else {
+          return ReadinessState.NOT_READY;
+        }
+      }
+
+      public List<Map<String, Object>> getResult() {
+        return docs;
+      }
+    };
+  }
+
+  @Override
+  public void setAdditionalProperties(Properties topologyProperties) {
+    topologyProperties.setProperty("es.clustername", "metron");
+    topologyProperties.setProperty("es.port", "9300");
+    topologyProperties.setProperty("es.ip", "localhost");
+    topologyProperties.setProperty("writer.class.name", "org.apache.metron.elasticsearch.writer.ElasticsearchWriter");
+  }
+
+  @Override
+  public String cleanField(String field) {
+    return field;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml
index 54c8fd5..84891e8 100644
--- a/metron-platform/metron-enrichment/pom.xml
+++ b/metron-platform/metron-enrichment/pom.xml
@@ -37,10 +37,25 @@
         </dependency>
         <dependency>
             <groupId>org.apache.metron</groupId>
+            <artifactId>metron-writer</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
             <artifactId>metron-hbase</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
         <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>${global_jackson_version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+            <version>${global_jackson_version}</version>
+        </dependency>
+        <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
             <version>${slf4j.version}</version>
@@ -65,7 +80,6 @@
                     <artifactId>log4j</artifactId>
                 </exclusion>
             </exclusions>
-            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
@@ -77,7 +91,6 @@
                     <groupId>javax.servlet</groupId>
                 </exclusion>
             </exclusions>
-            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.storm</groupId>
@@ -95,36 +108,36 @@
                 </exclusion>
             </exclusions>
         </dependency>
+
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${global_hbase_guava_version}</version>
+        </dependency>
         <dependency>
-            <groupId>org.apache.storm</groupId>
-            <artifactId>storm-hdfs</artifactId>
-            <version>${global_storm_version}</version>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka_2.9.2</artifactId>
+            <version>${global_kafka_version}</version>
             <exclusions>
                 <exclusion>
-                    <groupId>org.apache.storm</groupId>
-                    <artifactId>storm-core</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.hadoop</groupId>
-                    <artifactId>hadoop-client</artifactId>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
         <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>${global_guava_version}</version>
-        </dependency>
-        <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${global_hadoop_version}</version>
-            <scope>provided</scope>
             <exclusions>
                 <exclusion>
                     <artifactId>servlet-api</artifactId>
                     <groupId>javax.servlet</groupId>
                 </exclusion>
+                <exclusion>
+                    <artifactId>commons-httpclient</artifactId>
+                    <groupId>commons-httpclient</groupId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -144,9 +157,16 @@
             <version>${project.parent.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-integration-test</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
     <reporting>
         <plugins>
+
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
@@ -170,6 +190,7 @@
                     <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
                 </configuration>
             </plugin>
+
             <plugin>
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>emma-maven-plugin</artifactId>
@@ -188,6 +209,17 @@
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
                 <version>3.1</version>
                 <configuration>
@@ -214,10 +246,12 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <shadedArtifactAttached>true</shadedArtifactAttached>
+                            <shadedClassifierName>uber</shadedClassifierName>
                             <relocations>
                                 <relocation>
-                                    <pattern>com.google.common</pattern>
-                                    <shadedPattern>org.apache.metron.guava</shadedPattern>
+                                    <pattern>com.fasterxml.jackson</pattern>
+                                    <shadedPattern>org.apache.metron.jackson</shadedPattern>
                                 </relocation>
                             </relocations>
                             <transformers>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
index 6c9165c..4358370 100644
--- a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
@@ -53,5 +53,13 @@
       <fileMode>0644</fileMode>
       <lineEnding>unix</lineEnding>
     </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/target</directory>
+      <includes>
+        <include>${project.artifactId}-${project.version}-uber.jar</include>
+      </includes>
+      <outputDirectory>/lib</outputDirectory>
+      <useDefaultExcludes>true</useDefaultExcludes>
+    </fileSet>
   </fileSets>
 </assembly>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/config/enrichment.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment.properties b/metron-platform/metron-enrichment/src/main/config/enrichment.properties
new file mode 100644
index 0000000..e32915c
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/config/enrichment.properties
@@ -0,0 +1,82 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+##### Kafka #####
+
+kafka.zk=node1:2181
+kafka.broker=node1:6667
+enrichment.output.topic=indexing
+
+##### MySQL #####
+
+mysql.ip=10.22.0.214
+mysql.port=3306
+mysql.username=root
+mysql.password=hadoop123
+
+##### Metrics #####
+
+#reporters
+org.apache.metron.metrics.reporter.graphite=true
+org.apache.metron.metrics.reporter.console=false
+org.apache.metron.metrics.reporter.jmx=false
+
+#Graphite Addresses
+
+org.apache.metron.metrics.graphite.address=localhost
+org.apache.metron.metrics.graphite.port=2023
+
+#TelemetryParserBolt
+org.apache.metron.metrics.TelemetryParserBolt.acks=true
+org.apache.metron.metrics.TelemetryParserBolt.emits=true
+org.apache.metron.metrics.TelemetryParserBolt.fails=true
+
+
+#GenericEnrichmentBolt
+org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
+org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
+org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
+
+
+#TelemetryIndexingBolt
+org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
+org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
+org.apache.metron.metrics.TelemetryIndexingBolt.fails=true
+
+##### Host Enrichment #####
+
+org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
+{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
+{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
+
+
+##### HBase #####
+bolt.hbase.table.name=pcap
+bolt.hbase.table.fields=t:value
+bolt.hbase.table.key.tuple.field.name=key
+bolt.hbase.table.timestamp.tuple.field.name=timestamp
+bolt.hbase.enable.batching=false
+bolt.hbase.write.buffer.size.in.bytes=2000000
+bolt.hbase.durability=SKIP_WAL
+bolt.hbase.partitioner.region.info.refresh.interval.mins=60
+
+##### Threat Intel #####
+
+threat.intel.tracker.table=
+threat.intel.tracker.cf=
+threat.intel.ip.table=
+threat.intel.ip.cf=

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
index b499b24..69d5f66 100644
--- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
+++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
@@ -129,37 +129,16 @@ components:
                 args:
                     - ref: "simpleHBaseThreatIntelEnrichment"
 
-    -   id: "fileNameFormat"
-        className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat"
-        configMethods:
-            -   name: "withPrefix"
-                args:
-                    - "enrichment-"
-            -   name: "withExtension"
-                args:
-                  - ".json"
-            -   name: "withPath"
-                args:
-                    - "${index.hdfs.output}"
-
-    -   id: "hdfsRotationPolicy"
-        className: "${bolt.hdfs.rotation.policy}"
-        constructorArgs:
-          -  ${bolt.hdfs.rotation.policy.count}
-          - "${bolt.hdfs.rotation.policy.units}"
 #indexing
-    -   id: "hdfsWriter"
-        className: "org.apache.metron.writer.hdfs.HdfsWriter"
+    -   id: "kafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         configMethods:
-            -   name: "withFileNameFormat"
+            -   name: "withTopic"
                 args:
-                    - ref: "fileNameFormat"
-            -   name: "withRotationPolicy"
+                    - "${enrichment.output.topic}"
+            -   name: "withZkQuorum"
                 args:
-                    - ref: "hdfsRotationPolicy"
-
-    -   id: "indexWriter"
-        className: "${writer.class.name}"
+                    - "${kafka.zk}"
 
 #kafka/zookeeper
     -   id: "zkHosts"
@@ -277,22 +256,15 @@ bolts:
             -   name: "withMaxTimeRetain"
                 args: [10]
 # Indexing Bolts
-    -   id: "indexingBolt"
-        className: "org.apache.metron.enrichment.bolt.BulkMessageWriterBolt"
-        constructorArgs:
-            - "${kafka.zk}"
-        configMethods:
-            -   name: "withBulkMessageWriter"
-                args:
-                    - ref: "indexWriter"
-    -   id: "hdfsIndexingBolt"
-        className: "org.apache.metron.enrichment.bolt.BulkMessageWriterBolt"
+
+    -   id: "outputBolt"
+        className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
         constructorArgs:
             - "${kafka.zk}"
         configMethods:
-            -   name: "withBulkMessageWriter"
+            -   name: "withMessageWriter"
                 args:
-                    - ref: "hdfsWriter"
+                    - ref: "kafkaWriter"
 
 
 streams:
@@ -389,25 +361,12 @@ streams:
             streamId: "message"
             type: FIELDS
             args: ["key"]
-#indexing
-    -   name: "threatIntelJoin -> indexing"
+#output
+    -   name: "threatIntelJoin -> output"
         from: "threatIntelJoinBolt"
-        to: "indexingBolt"
+        to: "outputBolt"
         grouping:
             streamId: "message"
             type: FIELDS
             args: ["key"]
 
-    -   name: "threatIntelJoin -> hdfs"
-        from: "threatIntelJoinBolt"
-        to: "hdfsIndexingBolt"
-        grouping:
-            streamId: "message"
-            type: SHUFFLE
-
-    -   name: "indexingBolt -> errorIndexingBolt"
-        from: "indexingBolt"
-        to: "indexingBolt"
-        grouping:
-            streamId: "error"
-            type: SHUFFLE


[3/5] incubator-metron git commit: METRON-154: Decouple enrichment and indexing closes apache/incubator-metron#192

Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-indexing/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml
new file mode 100644
index 0000000..9760a11
--- /dev/null
+++ b/metron-platform/metron-indexing/pom.xml
@@ -0,0 +1,253 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 
+  Licensed to the Apache Software 
+	Foundation (ASF) under one or more contributor license agreements. See the 
+	NOTICE file distributed with this work for additional information regarding 
+	copyright ownership. The ASF licenses this file to You under the Apache License, 
+	Version 2.0 (the "License"); you may not use this file except in compliance 
+	with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+	Unless required by applicable law or agreed to in writing, software distributed 
+	under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES 
+	OR CONDITIONS OF ANY KIND, either express or implied. See the License for 
+  the specific language governing permissions and limitations under the License. 
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.metron</groupId>
+        <artifactId>metron-platform</artifactId>
+        <version>0.2.0BETA</version>
+    </parent>
+    <artifactId>metron-indexing</artifactId>
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <slf4j.version>1.7.7</slf4j.version>
+        <storm.hdfs.version>0.1.2</storm.hdfs.version>
+        <guava.version>${global_hbase_guava_version}</guava.version>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-common</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>${slf4j.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-hdfs</artifactId>
+            <version>${global_hadoop_version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>servlet-api</artifactId>
+                    <groupId>javax.servlet</groupId>
+                </exclusion>
+            </exclusions>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.storm</groupId>
+            <artifactId>storm-core</artifactId>
+            <version>${global_storm_version}</version>
+            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <artifactId>servlet-api</artifactId>
+                    <groupId>javax.servlet</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>log4j-over-slf4j</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.storm</groupId>
+            <artifactId>storm-hdfs</artifactId>
+            <version>${global_storm_version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.storm</groupId>
+                    <artifactId>storm-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-client</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${global_guava_version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>${global_hadoop_version}</version>
+            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <artifactId>servlet-api</artifactId>
+                    <groupId>javax.servlet</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-all</artifactId>
+            <version>${global_mockito_version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-validator</groupId>
+            <artifactId>commons-validator</artifactId>
+            <version>1.4.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-test-utilities</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-integration-test</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-enrichment</artifactId>
+            <version>${project.parent.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+    <reporting>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <systemProperties>
+                        <property>
+                            <name>mode</name>
+                            <value>global</value>
+                        </property>
+                    </systemProperties>
+                </configuration>
+            </plugin>
+
+            <!-- Normally, dependency report takes time, skip it -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-project-info-reports-plugin</artifactId>
+                <version>2.7</version>
+
+                <configuration>
+                    <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>emma-maven-plugin</artifactId>
+                <version>1.0-alpha-3</version>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-pmd-plugin</artifactId>
+                <configuration>
+                  <targetJdk>${global_java_version}</targetJdk>
+                </configuration>
+            </plugin>
+        </plugins>
+    </reporting>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.1</version>
+                <configuration>
+                  <source>${global_java_version}</source>
+                  <target>${global_java_version}</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>${global_shade_version}</version>
+                <configuration>
+                    <createDependencyReducedPom>true</createDependencyReducedPom>
+                    <artifactSet>
+                        <excludes>
+                            <exclude>*slf4j*</exclude>
+                        </excludes>
+                    </artifactSet>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <relocations>
+                                <relocation>
+                                    <pattern>com.google.common</pattern>
+                                    <shadedPattern>org.apache.metron.guava</shadedPattern>
+                                </relocation>
+                            </relocations>
+                            <transformers>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+                                    <resource>.yaml</resource>
+                                </transformer>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass></mainClass>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptor>src/main/assembly/assembly.xml</descriptor>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id> <!-- this is used for inheritance merges -->
+                        <phase>package</phase> <!-- bind to the packaging phase -->
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-indexing/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/assembly/assembly.xml b/metron-platform/metron-indexing/src/main/assembly/assembly.xml
new file mode 100644
index 0000000..6c9165c
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/assembly/assembly.xml
@@ -0,0 +1,57 @@
+<!--
+  Licensed to the Apache Software
+	Foundation (ASF) under one or more contributor license agreements. See the
+	NOTICE file distributed with this work for additional information regarding
+	copyright ownership. The ASF licenses this file to You under the Apache License,
+	Version 2.0 (the "License"); you may not use this file except in compliance
+	with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+	Unless required by applicable law or agreed to in writing, software distributed
+	under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+	OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+  the specific language governing permissions and limitations under the License.
+  -->
+
+<assembly>
+  <id>archive</id>
+  <formats>
+    <format>tar.gz</format>
+  </formats>
+  <includeBaseDirectory>false</includeBaseDirectory>
+  <fileSets>
+    <fileSet>
+      <directory>${project.basedir}/src/main/config</directory>
+      <outputDirectory>/config</outputDirectory>
+      <useDefaultExcludes>true</useDefaultExcludes>
+      <excludes>
+        <exclude>**/*.formatted</exclude>
+        <exclude>**/*.filtered</exclude>
+      </excludes>
+      <fileMode>0644</fileMode>
+      <lineEnding>unix</lineEnding>
+      <filtered>true</filtered>
+    </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/src/main/scripts</directory>
+      <outputDirectory>/bin</outputDirectory>
+      <useDefaultExcludes>true</useDefaultExcludes>
+      <excludes>
+        <exclude>**/*.formatted</exclude>
+        <exclude>**/*.filtered</exclude>
+      </excludes>
+      <fileMode>0755</fileMode>
+      <lineEnding>unix</lineEnding>
+      <filtered>true</filtered>
+    </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/src/main/flux</directory>
+      <outputDirectory>/flux</outputDirectory>
+      <useDefaultExcludes>true</useDefaultExcludes>
+      <excludes>
+        <exclude>**/*.formatted</exclude>
+        <exclude>**/*.filtered</exclude>
+      </excludes>
+      <fileMode>0644</fileMode>
+      <lineEnding>unix</lineEnding>
+    </fileSet>
+  </fileSets>
+</assembly>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml b/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml
new file mode 100644
index 0000000..d1e697b
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml
@@ -0,0 +1,152 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "indexing"
+config:
+    topology.workers: ${indexing.workers}
+    topology.acker.executors: ${indexing.executors}
+
+components:
+
+    -   id: "fileNameFormat"
+        className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat"
+        configMethods:
+            -   name: "withPrefix"
+                args:
+                    - "enrichment-"
+            -   name: "withExtension"
+                args:
+                  - ".json"
+            -   name: "withPath"
+                args:
+                    - "${index.hdfs.output}"
+
+    -   id: "hdfsRotationPolicy"
+        className: "${bolt.hdfs.rotation.policy}"
+        constructorArgs:
+          -  ${bolt.hdfs.rotation.policy.count}
+          - "${bolt.hdfs.rotation.policy.units}"
+#indexing
+    -   id: "hdfsWriter"
+        className: "org.apache.metron.writer.hdfs.HdfsWriter"
+        configMethods:
+            -   name: "withFileNameFormat"
+                args:
+                    - ref: "fileNameFormat"
+            -   name: "withRotationPolicy"
+                args:
+                    - ref: "hdfsRotationPolicy"
+    -   id: "kafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
+        configMethods:
+            -   name: "withTopic"
+                args:
+                    - "${index.error.topic}"
+            -   name: "withZkQuorum"
+                args:
+                    - "${kafka.zk}"
+
+    -   id: "indexWriter"
+        className: "${writer.class.name}"
+
+#kafka/zookeeper
+    -   id: "zkHosts"
+        className: "storm.kafka.ZkHosts"
+        constructorArgs:
+            - "${kafka.zk}"
+    -   id: "kafkaConfig"
+        className: "org.apache.metron.common.spout.kafka.SpoutConfig"
+        constructorArgs:
+            # zookeeper hosts
+            - ref: "zkHosts"
+            # topic name
+            - "${index.input.topic}"
+            # zk root
+            - ""
+            # id
+            - "indexing"
+        configMethods:
+            -   name: "from"
+                args:
+                    - "${kafka.start}"
+
+spouts:
+    -   id: "kafkaSpout"
+        className: "storm.kafka.KafkaSpout"
+        constructorArgs:
+            - ref: "kafkaConfig"
+bolts:
+
+# Indexing Bolts
+    -   id: "indexingBolt"
+        className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withBulkMessageWriter"
+                args:
+                    - ref: "indexWriter"
+            -   name: "withMessageGetter"
+                args:
+                    - "RAW"
+    -   id: "hdfsIndexingBolt"
+        className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withBulkMessageWriter"
+                args:
+                    - ref: "hdfsWriter"
+            -   name: "withMessageGetter"
+                args:
+                    - "RAW"
+
+    -   id: "indexingErrorBolt"
+        className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withMessageWriter"
+                args:
+                    - ref: "kafkaWriter"
+
+streams:
+
+    -   name: "spout -> indexing"
+        from: "kafkaSpout"
+        to: "indexingBolt"
+        grouping:
+            type: SHUFFLE
+
+    -   name: "spout -> hdfs"
+        from: "kafkaSpout"
+        to: "hdfsIndexingBolt"
+        grouping:
+            type: SHUFFLE
+
+    -   name: "indexingBolt -> errorIndexingBolt"
+        from: "indexingBolt"
+        to: "indexingErrorBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+
+    -   name: "hdfsBolt -> errorIndexingBolt"
+        from: "hdfsIndexingBolt"
+        to: "indexingErrorBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
new file mode 100644
index 0000000..2c85308
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
@@ -0,0 +1,240 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.integration;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import org.apache.metron.TestConstants;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.interfaces.FieldNameConverter;
+import org.apache.metron.common.spout.kafka.SpoutConfig;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
+import org.apache.metron.integration.BaseIntegrationTest;
+import org.apache.metron.integration.ComponentRunner;
+import org.apache.metron.integration.InMemoryComponent;
+import org.apache.metron.integration.Processor;
+import org.apache.metron.integration.components.FluxTopologyComponent;
+import org.apache.metron.integration.components.KafkaWithZKComponent;
+import org.apache.metron.integration.utils.TestUtils;
+import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy;
+import org.junit.Assert;
+import org.junit.Test;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+
+public abstract class IndexingIntegrationTest extends BaseIntegrationTest {
+  protected String hdfsDir = "target/indexingIntegrationTest/hdfs";
+  protected String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
+  protected String fluxPath = "../metron-indexing/src/main/flux/indexing/remote.yaml";
+  protected String testSensorType = "test";
+
+
+  public static List<Map<String, Object>> readDocsFromDisk(String hdfsDirStr) throws IOException {
+    List<Map<String, Object>> ret = new ArrayList<>();
+    File hdfsDir = new File(hdfsDirStr);
+    Stack<File> fs = new Stack<>();
+    if (hdfsDir.exists()) {
+      fs.push(hdfsDir);
+      while (!fs.empty()) {
+        File f = fs.pop();
+        if (f.isDirectory()) {
+          for (File child : f.listFiles()) {
+            fs.push(child);
+          }
+        } else {
+          System.out.println("Processed " + f);
+          if (f.getName().startsWith("enrichment") || f.getName().endsWith(".json")) {
+            List<byte[]> data = TestUtils.readSampleData(f.getPath());
+            Iterables.addAll(ret, Iterables.transform(data, new Function<byte[], Map<String, Object>>() {
+              @Nullable
+              @Override
+              public Map<String, Object> apply(@Nullable byte[] bytes) {
+                String s = new String(bytes);
+                try {
+                  return JSONUtils.INSTANCE.load(s, new TypeReference<Map<String, Object>>() {
+                  });
+                } catch (IOException e) {
+                  throw new RuntimeException(e);
+                }
+              }
+            }));
+          }
+        }
+      }
+    }
+    return ret;
+  }
+
+  public static void cleanHdfsDir(String hdfsDirStr) {
+    File hdfsDir = new File(hdfsDirStr);
+    Stack<File> fs = new Stack<>();
+    if (hdfsDir.exists()) {
+      fs.push(hdfsDir);
+      while (!fs.empty()) {
+        File f = fs.pop();
+        if (f.isDirectory()) {
+          for (File child : f.listFiles()) {
+            fs.push(child);
+          }
+        } else {
+          if (f.getName().startsWith("enrichment") || f.getName().endsWith(".json")) {
+            f.delete();
+          }
+        }
+      }
+    }
+  }
+
+  @Test
+  public void test() throws Exception {
+    cleanHdfsDir(hdfsDir);
+    final String dateFormat = "yyyy.MM.dd.HH";
+    final List<byte[]> inputMessages = TestUtils.readSampleData(sampleParsedPath);
+    final Properties topologyProperties = new Properties() {{
+      setProperty("kafka.start", SpoutConfig.Offset.BEGINNING.name());
+      setProperty("indexing.workers", "1");
+      setProperty("indexing.executors", "0");
+      setProperty("index.input.topic", Constants.INDEXING_TOPIC);
+      setProperty("index.error.topic", Constants.INDEXING_ERROR_TOPIC);
+      setProperty("index.date.format", dateFormat);
+      //HDFS settings
+
+      setProperty("bolt.hdfs.rotation.policy", TimedRotationPolicy.class.getCanonicalName());
+      setProperty("bolt.hdfs.rotation.policy.count", "1");
+      setProperty("bolt.hdfs.rotation.policy.units", "DAYS");
+      setProperty("index.hdfs.output", hdfsDir);
+    }};
+    setAdditionalProperties(topologyProperties);
+    final KafkaWithZKComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaWithZKComponent.Topic>() {{
+      add(new KafkaWithZKComponent.Topic(Constants.INDEXING_TOPIC, 1));
+      add(new KafkaWithZKComponent.Topic(Constants.INDEXING_ERROR_TOPIC, 1));
+    }});
+    List<Map<String, Object>> inputDocs = new ArrayList<>();
+    for(byte[] b : inputMessages) {
+      Map<String, Object> m = JSONUtils.INSTANCE.load(new String(b), new TypeReference<Map<String, Object>>() {});
+      inputDocs.add(m);
+
+    }
+    ConfigUploadComponent configUploadComponent = new ConfigUploadComponent()
+            .withTopologyProperties(topologyProperties)
+            .withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH)
+            .withEnrichmentConfigsPath(TestConstants.SAMPLE_CONFIG_PATH);
+    FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder()
+            .withTopologyLocation(new File(fluxPath))
+            .withTopologyName("test")
+            .withTopologyProperties(topologyProperties)
+            .build();
+
+
+    ComponentRunner runner = new ComponentRunner.Builder()
+            .withComponent("kafka", kafkaComponent)
+            .withComponent("config", configUploadComponent)
+            .withComponent("storm", fluxComponent)
+            .withComponent("search", getSearchComponent(topologyProperties))
+            .withMillisecondsBetweenAttempts(15000)
+            .withNumRetries(10)
+            .build();
+    runner.start();
+
+    try {
+      fluxComponent.submitTopology();
+
+      kafkaComponent.writeMessages(Constants.INDEXING_TOPIC, inputMessages);
+      List<Map<String, Object>> docs = cleanDocs(runner.process(getProcessor(inputMessages)));
+      Assert.assertEquals(docs.size(), inputMessages.size());
+      //assert that our input docs are equivalent to the output docs, converting the input docs keys based
+      // on the field name converter
+      assertInputDocsMatchOutputs(inputDocs, docs, getFieldNameConverter());
+      assertInputDocsMatchOutputs(inputDocs, readDocsFromDisk(hdfsDir), x -> x);
+    }
+    finally {
+      if(runner != null) {
+        runner.stop();
+      }
+    }
+  }
+
+  public List<Map<String, Object>> cleanDocs(List<Map<String, Object>> docs) {
+    List<Map<String, Object>> ret = new ArrayList<>();
+    for (Map<String, Object> doc : docs) {
+      Map<String, Object> msg = new HashMap<>();
+      for (Map.Entry<String, Object> kv : doc.entrySet()) {
+        //for writers like solr who modify the keys, we want to undo that if we can
+        msg.put(cleanField(kv.getKey()), kv.getValue());
+      }
+      ret.add(msg);
+    }
+    return ret;
+  }
+
+  public void assertInputDocsMatchOutputs( List<Map<String, Object>> inputDocs
+                                         , List<Map<String, Object>> indexDocs
+                                         , FieldNameConverter converter
+                                         )
+  {
+    for(Map<String, Object> indexDoc : indexDocs) {
+      boolean foundMatch = false;
+      for(Map<String, Object> doc : inputDocs) {
+        if(docMatches(indexDoc, doc, converter)) {
+          foundMatch = true;
+          break;
+        }
+      }
+      if(!foundMatch) {
+        System.err.println("Unable to find: ");
+        printMessage(indexDoc);
+        dumpMessages("INPUT DOCS:", inputDocs);
+      }
+      Assert.assertTrue(foundMatch);
+    }
+  }
+
+  private void printMessage(Map<String, Object> doc) {
+    TreeMap<String, Object> d = new TreeMap<>(doc);
+      for(Map.Entry<String, Object> kv : d.entrySet()) {
+        System.err.println("  " + kv.getKey() + " -> " + kv.getValue());
+      }
+  }
+
+  private void dumpMessages(String title, List<Map<String, Object>> docs) {
+    System.err.println(title);
+    int cnt = 0;
+    for(Map<String, Object> doc : docs) {
+      System.err.println("MESSAGE " + cnt++);
+      printMessage(doc);
+    }
+  }
+
+  boolean docMatches(Map<String, Object> indexedDoc, Map<String, Object> inputDoc, FieldNameConverter converter) {
+    String key = "original_string";
+    String indexKey = converter.convert(key);
+    String originalString = inputDoc.get(key).toString();
+    return originalString.equals(indexedDoc.get(indexKey).toString());
+  }
+  public abstract Processor<List<Map<String, Object>>> getProcessor (List <byte[]>inputMessages);
+  public abstract FieldNameConverter getFieldNameConverter();
+  public abstract InMemoryComponent getSearchComponent(final Properties topologyProperties) throws Exception;
+  public abstract void setAdditionalProperties(Properties topologyProperties);
+  public abstract String cleanField(String field);
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml
index 130de5e..4aa10cf 100644
--- a/metron-platform/metron-integration-test/pom.xml
+++ b/metron-platform/metron-integration-test/pom.xml
@@ -143,11 +143,6 @@
     </dependency>
     <dependency>
       <groupId>org.apache.metron</groupId>
-      <artifactId>metron-enrichment</artifactId>
-      <version>${project.parent.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.metron</groupId>
       <artifactId>metron-test-utilities</artifactId>
       <version>${project.parent.version}</version>
     </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java
index 1a3247c..5fdae49 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java
@@ -20,7 +20,6 @@ package org.apache.metron.integration;
 import com.google.common.base.Function;
 import org.apache.metron.TestConstants;
 import org.apache.metron.integration.components.KafkaWithZKComponent;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
 
 import javax.annotation.Nullable;
 import java.util.List;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
deleted file mode 100644
index 483f2b5..0000000
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
+++ /dev/null
@@ -1,489 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.google.common.base.*;
-
-import com.google.common.collect.Iterables;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.metron.common.Constants;
-import org.apache.metron.TestConstants;
-import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.common.interfaces.FieldNameConverter;
-import org.apache.metron.common.configuration.EnrichmentConfigurations;
-import org.apache.metron.hbase.TableProvider;
-import org.apache.metron.enrichment.converter.EnrichmentKey;
-import org.apache.metron.enrichment.converter.EnrichmentValue;
-import org.apache.metron.enrichment.converter.EnrichmentHelper;
-import org.apache.metron.integration.components.ConfigUploadComponent;
-import org.apache.metron.integration.utils.TestUtils;
-import org.apache.metron.test.utils.UnitTestHelper;
-import org.apache.metron.integration.components.FluxTopologyComponent;
-import org.apache.metron.integration.components.KafkaWithZKComponent;
-import org.apache.metron.integration.mock.MockGeoAdapter;
-import org.apache.metron.test.mock.MockHTable;
-import org.apache.metron.enrichment.lookup.LookupKV;
-
-import org.apache.metron.integration.utils.SampleUtil;
-import org.apache.metron.common.utils.JSONUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-import javax.annotation.Nullable;
-import java.io.File;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.Stack;
-
-public abstract class EnrichmentIntegrationTest extends BaseIntegrationTest {
-
-  private static final String SRC_IP = "ip_src_addr";
-  private static final String DST_IP = "ip_dst_addr";
-  private static final String MALICIOUS_IP_TYPE = "malicious_ip";
-  private static final String PLAYFUL_CLASSIFICATION_TYPE = "playful_classification";
-  private static final Map<String, Object> PLAYFUL_ENRICHMENT = new HashMap<String, Object>() {{
-    put("orientation", "north");
-  }};
-  protected String testSensorType = "test";
-  protected String hdfsDir = "target/enrichmentIntegrationTest/hdfs";
-  protected String fluxPath = "../metron-enrichment/src/main/flux/enrichment/test.yaml";
-  protected String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
-  private String sampleIndexedPath = TestConstants.SAMPLE_DATA_INDEXED_PATH + "TestIndexed";
-
-  public static class Provider implements TableProvider, Serializable {
-    MockHTable.Provider  provider = new MockHTable.Provider();
-    @Override
-    public HTableInterface getTable(Configuration config, String tableName) throws IOException {
-      return provider.getTable(config, tableName);
-    }
-  }
-
-  public static void cleanHdfsDir(String hdfsDirStr) {
-    File hdfsDir = new File(hdfsDirStr);
-    Stack<File> fs = new Stack<>();
-    if(hdfsDir.exists()) {
-      fs.push(hdfsDir);
-      while(!fs.empty()) {
-        File f = fs.pop();
-        if (f.isDirectory()) {
-          for(File child : f.listFiles()) {
-            fs.push(child);
-          }
-        }
-        else {
-          if (f.getName().startsWith("enrichment") || f.getName().endsWith(".json")) {
-            f.delete();
-          }
-        }
-      }
-    }
-  }
-
-  public static List<Map<String, Object> > readDocsFromDisk(String hdfsDirStr) throws IOException {
-    List<Map<String, Object>> ret = new ArrayList<>();
-    File hdfsDir = new File(hdfsDirStr);
-    Stack<File> fs = new Stack<>();
-    if(hdfsDir.exists()) {
-      fs.push(hdfsDir);
-      while(!fs.empty()) {
-        File f = fs.pop();
-        if(f.isDirectory()) {
-          for (File child : f.listFiles()) {
-            fs.push(child);
-          }
-        }
-        else {
-          System.out.println("Processed " + f);
-          if (f.getName().startsWith("enrichment") || f.getName().endsWith(".json")) {
-            List<byte[]> data = TestUtils.readSampleData(f.getPath());
-            Iterables.addAll(ret, Iterables.transform(data, new Function<byte[], Map<String, Object>>() {
-              @Nullable
-              @Override
-              public Map<String, Object> apply(@Nullable byte[] bytes) {
-                String s = new String(bytes);
-                try {
-                  return JSONUtils.INSTANCE.load(s, new TypeReference<Map<String, Object>>() {
-                  });
-                } catch (IOException e) {
-                  throw new RuntimeException(e);
-                }
-              }
-            }));
-          }
-        }
-      }
-    }
-    return ret;
-  }
-
-
-  @Test
-  public void test() throws Exception {
-    cleanHdfsDir(hdfsDir);
-    final EnrichmentConfigurations configurations = SampleUtil.getSampleEnrichmentConfigs();
-    final String dateFormat = "yyyy.MM.dd.HH";
-    final List<byte[]> inputMessages = TestUtils.readSampleData(sampleParsedPath);
-    final String cf = "cf";
-    final String trackerHBaseTableName = "tracker";
-    final String threatIntelTableName = "threat_intel";
-    final String enrichmentsTableName = "enrichments";
-    final Properties topologyProperties = new Properties() {{
-      setProperty("org.apache.metron.enrichment.host.known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"},\n" +
-              "{\"ip\":\"10.1.128.237\", \"local\":\"UNKNOWN\", \"type\":\"unknown\", \"asset_value\" : \"important\"},\n" +
-              "{\"ip\":\"10.60.10.254\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"},\n" +
-              "{\"ip\":\"10.0.2.15\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}]");
-      setProperty("hbase.provider.impl","" + Provider.class.getName());
-      setProperty("threat.intel.tracker.table", trackerHBaseTableName);
-      setProperty("threat.intel.tracker.cf", cf);
-      setProperty("threat.intel.simple.hbase.table", threatIntelTableName);
-      setProperty("threat.intel.simple.hbase.cf", cf);
-      setProperty("enrichment.simple.hbase.table", enrichmentsTableName);
-      setProperty("enrichment.simple.hbase.cf", cf);
-      setProperty("es.clustername", "metron");
-      setProperty("es.port", "9300");
-      setProperty("es.ip", "localhost");
-      setProperty("index.date.format", dateFormat);
-      setProperty("index.hdfs.output", hdfsDir);
-    }};
-    setAdditionalProperties(topologyProperties);
-    final KafkaWithZKComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaWithZKComponent.Topic>() {{
-      add(new KafkaWithZKComponent.Topic(Constants.ENRICHMENT_TOPIC, 1));
-    }});
-
-    ConfigUploadComponent configUploadComponent = new ConfigUploadComponent()
-            .withTopologyProperties(topologyProperties)
-            .withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH)
-            .withEnrichmentConfigsPath(TestConstants.SAMPLE_CONFIG_PATH);
-
-    //create MockHBaseTables
-    final MockHTable trackerTable = (MockHTable)MockHTable.Provider.addToCache(trackerHBaseTableName, cf);
-    final MockHTable threatIntelTable = (MockHTable)MockHTable.Provider.addToCache(threatIntelTableName, cf);
-    EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>(){{
-      add(new LookupKV<>(new EnrichmentKey(MALICIOUS_IP_TYPE, "10.0.2.3"), new EnrichmentValue(new HashMap<>())));
-    }});
-    final MockHTable enrichmentTable = (MockHTable)MockHTable.Provider.addToCache(enrichmentsTableName, cf);
-    EnrichmentHelper.INSTANCE.load(enrichmentTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>(){{
-      add(new LookupKV<>(new EnrichmentKey(PLAYFUL_CLASSIFICATION_TYPE, "10.0.2.3")
-                        , new EnrichmentValue(PLAYFUL_ENRICHMENT )
-                        )
-         );
-    }});
-    FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder()
-            .withTopologyLocation(new File(fluxPath))
-            .withTopologyName("test")
-            .withTopologyProperties(topologyProperties)
-            .build();
-
-    InMemoryComponent searchComponent = getSearchComponent(topologyProperties);
-
-    UnitTestHelper.verboseLogging();
-    ComponentRunner runner = new ComponentRunner.Builder()
-            .withComponent("kafka", kafkaComponent)
-            .withComponent("config", configUploadComponent)
-            .withComponent("search", searchComponent)
-            .withComponent("storm", fluxComponent)
-            .withMillisecondsBetweenAttempts(15000)
-            .withNumRetries(10)
-            .build();
-    runner.start();
-
-    try {
-      fluxComponent.submitTopology();
-
-      kafkaComponent.writeMessages(Constants.ENRICHMENT_TOPIC, inputMessages);
-      List<Map<String, Object>> docs = runner.process(getProcessor(inputMessages));
-      Assert.assertEquals(inputMessages.size(), docs.size());
-      List<Map<String, Object>> cleanedDocs = cleanDocs(docs);
-      validateAll(cleanedDocs, getFieldNameConverter());
-
-
-      List<Map<String, Object>> docsFromDisk = readDocsFromDisk(hdfsDir);
-      Assert.assertEquals(docsFromDisk.size(), docs.size()) ;
-      Assert.assertEquals(new File(hdfsDir).list().length, 1);
-      Assert.assertEquals(new File(hdfsDir).list()[0], testSensorType);
-      //we want the identity transformation when dealing with docs on disk.
-      validateAll(docsFromDisk, fieldName -> fieldName);
-    }
-    finally {
-      cleanHdfsDir(hdfsDir);
-      runner.stop();
-    }
-  }
-
-  public List<Map<String, Object>> cleanDocs(List<Map<String, Object>> docs) {
-    List<Map<String, Object>> cleanedDocs = new ArrayList<>();
-    for(Map<String, Object> doc: docs) {
-      Map<String, Object> cleanedFields = new HashMap<>();
-      for(String field: doc.keySet()) {
-        cleanedFields.put(cleanField(field), doc.get(field));
-      }
-      cleanedDocs.add(cleanedFields);
-    }
-    return cleanedDocs;
-  }
-
-  public static void validateAll(List<Map<String, Object>> docs, FieldNameConverter fnc) {
-
-    for (Map<String, Object> doc : docs) {
-      baseValidation(doc, fnc);
-      hostEnrichmentValidation(doc, fnc);
-      geoEnrichmentValidation(doc, fnc);
-      threatIntelValidation(doc, fnc);
-      simpleEnrichmentValidation(doc, fnc);
-    }
-  }
-
-  public static void baseValidation(Map<String, Object> jsonDoc, FieldNameConverter fnc) {
-    assertEnrichmentsExists("threatintels.", setOf("hbaseThreatIntel"), jsonDoc.keySet());
-    assertEnrichmentsExists("enrichments.", setOf("geo", "host", "hbaseEnrichment" ), jsonDoc.keySet());
-    for(Map.Entry<String, Object> kv : jsonDoc.entrySet()) {
-      //ensure no values are empty.
-      Assert.assertTrue(kv.getValue().toString().length() > 0);
-    }
-    //ensure we always have a source ip and destination ip
-    Assert.assertNotNull(jsonDoc.get(SRC_IP));
-    Assert.assertNotNull(jsonDoc.get(DST_IP));
-  }
-
-  private static class EvaluationPayload {
-    Map<String, Object> indexedDoc;
-    String key;
-    FieldNameConverter fnc;
-    public EvaluationPayload(Map<String, Object> indexedDoc, String key, FieldNameConverter fnc) {
-      this.indexedDoc = indexedDoc;
-      this.key = key;
-      this.fnc = fnc;
-    }
-  }
-
-  private static enum HostEnrichments implements Predicate<EvaluationPayload>{
-
-    LOCAL_LOCATION(new Predicate<EvaluationPayload>() {
-
-      @Override
-      public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
-
-        return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.local"),"").equals("YES");
-
-      }
-    })
-
-    ,UNKNOWN_LOCATION(new Predicate<EvaluationPayload>() {
-
-      @Override
-      public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
-        return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.local"),"").equals("UNKNOWN");
-      }
-    })
-    ,IMPORTANT(new Predicate<EvaluationPayload>() {
-      @Override
-      public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
-        return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.asset_value"),"").equals("important");
-      }
-    })
-    ,PRINTER_TYPE(new Predicate<EvaluationPayload>() {
-      @Override
-      public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
-        return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.type"),"").equals("printer");
-      }
-    })
-    ,WEBSERVER_TYPE(new Predicate<EvaluationPayload>() {
-      @Override
-      public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
-        return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.type"),"").equals("webserver");
-      }
-    })
-    ,UNKNOWN_TYPE(new Predicate<EvaluationPayload>() {
-      @Override
-      public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
-        return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.type"),"").equals("unknown");
-      }
-    })
-    ;
-
-    Predicate<EvaluationPayload> _predicate;
-    HostEnrichments(Predicate<EvaluationPayload> predicate) {
-      this._predicate = predicate;
-    }
-
-    public boolean apply(EvaluationPayload payload) {
-      return _predicate.apply(payload);
-    }
-
-  }
-
-  private static void assertEnrichmentsExists(String topLevel, Set<String> expectedEnrichments, Set<String> keys) {
-    for(String key : keys) {
-      if(key.startsWith(topLevel)) {
-        String secondLevel = Iterables.get(Splitter.on(".").split(key), 1);
-        String message = "Found an enrichment/threat intel (" + secondLevel + ") that I didn't expect (expected enrichments :"
-                       + Joiner.on(",").join(expectedEnrichments) + "), but it was not there.  If you've created a new"
-                       + " enrichment, then please add a validation method to this unit test.  Otherwise, it's a solid error"
-                       + " and should be investigated.";
-        Assert.assertTrue( message, expectedEnrichments.contains(secondLevel));
-      }
-    }
-  }
-  private static void simpleEnrichmentValidation(Map<String, Object> indexedDoc, FieldNameConverter fnc) {
-    if(indexedDoc.getOrDefault(fnc.convert(SRC_IP),"").equals("10.0.2.3")
-            || indexedDoc.getOrDefault(fnc.convert(DST_IP),"").equals("10.0.2.3")
-            ) {
-      Assert.assertTrue(keyPatternExists(fnc.convert("enrichments.hbaseEnrichment"), indexedDoc));
-      if(indexedDoc.getOrDefault(fnc.convert(SRC_IP),"").equals("10.0.2.3")) {
-        Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.hbaseEnrichment." + SRC_IP + "." + PLAYFUL_CLASSIFICATION_TYPE+ ".orientation"))
-                , PLAYFUL_ENRICHMENT.get("orientation")
-        );
-      }
-      else if(indexedDoc.getOrDefault(fnc.convert(DST_IP),"").equals("10.0.2.3")) {
-        Assert.assertEquals( indexedDoc.get(fnc.convert("enrichments.hbaseEnrichment." + DST_IP + "." + PLAYFUL_CLASSIFICATION_TYPE + ".orientation"))
-                , PLAYFUL_ENRICHMENT.get("orientation")
-        );
-      }
-    }
-
-  }
-  private static void threatIntelValidation(Map<String, Object> indexedDoc, FieldNameConverter fnc) {
-    if(indexedDoc.getOrDefault(fnc.convert(SRC_IP),"").equals("10.0.2.3")
-    || indexedDoc.getOrDefault(fnc.convert(DST_IP),"").equals("10.0.2.3")
-            ) {
-      //if we have any threat intel messages, we want to tag is_alert to true
-      Assert.assertTrue(keyPatternExists(fnc.convert("threatintels."), indexedDoc));
-      Assert.assertTrue(indexedDoc.containsKey(fnc.convert("threat.triage.level")));
-      Assert.assertEquals(indexedDoc.getOrDefault(fnc.convert("is_alert"),""), "true");
-      Assert.assertEquals((double)indexedDoc.get(fnc.convert("threat.triage.level")), 10d, 1e-7);
-    }
-    else {
-      //For YAF this is the case, but if we do snort later on, this will be invalid.
-      Assert.assertNull(indexedDoc.get(fnc.convert("is_alert")));
-      Assert.assertFalse(keyPatternExists(fnc.convert("threatintels."), indexedDoc));
-    }
-    //ip threat intels
-    if(keyPatternExists(fnc.convert("threatintels.hbaseThreatIntel."), indexedDoc)) {
-      if(indexedDoc.getOrDefault(fnc.convert(SRC_IP),"").equals("10.0.2.3")) {
-        Assert.assertEquals(indexedDoc.get(fnc.convert("threatintels.hbaseThreatIntel." + SRC_IP + "." + MALICIOUS_IP_TYPE)), "alert");
-      }
-      else if(indexedDoc.getOrDefault(fnc.convert(DST_IP),"").equals("10.0.2.3")) {
-        Assert.assertEquals(indexedDoc.get(fnc.convert("threatintels.hbaseThreatIntel." + DST_IP + "." + MALICIOUS_IP_TYPE)), "alert");
-      }
-      else {
-        Assert.fail("There was a threat intels that I did not expect: " + indexedDoc);
-      }
-    }
-
-  }
-
-  private static void geoEnrichmentValidation(Map<String, Object> indexedDoc, FieldNameConverter fnc) {
-    //should have geo enrichment on every message due to mock geo adapter
-    Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".location_point")), MockGeoAdapter.DEFAULT_LOCATION_POINT);
-    Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP +".location_point")), MockGeoAdapter.DEFAULT_LOCATION_POINT);
-    Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".longitude")), MockGeoAdapter.DEFAULT_LONGITUDE);
-    Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".longitude")), MockGeoAdapter.DEFAULT_LONGITUDE);
-    Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".city")), MockGeoAdapter.DEFAULT_CITY);
-    Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".city")), MockGeoAdapter.DEFAULT_CITY);
-    Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".latitude")), MockGeoAdapter.DEFAULT_LATITUDE);
-    Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".latitude")), MockGeoAdapter.DEFAULT_LATITUDE);
-    Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".country")), MockGeoAdapter.DEFAULT_COUNTRY);
-    Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".country")), MockGeoAdapter.DEFAULT_COUNTRY);
-    Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".dmaCode")), MockGeoAdapter.DEFAULT_DMACODE);
-    Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".dmaCode")), MockGeoAdapter.DEFAULT_DMACODE);
-    Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".postalCode")), MockGeoAdapter.DEFAULT_POSTAL_CODE);
-    Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".postalCode")), MockGeoAdapter.DEFAULT_POSTAL_CODE);
-  }
-
-  private static void hostEnrichmentValidation(Map<String, Object> indexedDoc, FieldNameConverter fnc) {
-    boolean enriched = false;
-    //important local printers
-    {
-      Set<String> ips = setOf("10.0.2.15", "10.60.10.254");
-      if (ips.contains(indexedDoc.get(fnc.convert(SRC_IP)))) {
-        //this is a local, important, printer
-        Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
-                ,HostEnrichments.IMPORTANT
-                ,HostEnrichments.PRINTER_TYPE
-                ).apply(new EvaluationPayload(indexedDoc, SRC_IP, fnc))
-        );
-        enriched = true;
-      }
-      if (ips.contains(indexedDoc.get(fnc.convert(DST_IP)))) {
-        Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
-                ,HostEnrichments.IMPORTANT
-                ,HostEnrichments.PRINTER_TYPE
-                ).apply(new EvaluationPayload(indexedDoc, DST_IP, fnc))
-        );
-        enriched = true;
-      }
-    }
-    //important local webservers
-    {
-      Set<String> ips = setOf("10.1.128.236");
-      if (ips.contains(indexedDoc.get(fnc.convert(SRC_IP)))) {
-        //this is a local, important, printer
-        Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
-                ,HostEnrichments.IMPORTANT
-                ,HostEnrichments.WEBSERVER_TYPE
-                ).apply(new EvaluationPayload(indexedDoc, SRC_IP, fnc))
-        );
-        enriched = true;
-      }
-      if (ips.contains(indexedDoc.get(fnc.convert(DST_IP)))) {
-        Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
-                ,HostEnrichments.IMPORTANT
-                ,HostEnrichments.WEBSERVER_TYPE
-                ).apply(new EvaluationPayload(indexedDoc, DST_IP, fnc))
-        );
-        enriched = true;
-      }
-    }
-    if(!enriched) {
-      Assert.assertFalse(keyPatternExists("enrichments.host", indexedDoc));
-    }
-  }
-
-
-  private static boolean keyPatternExists(String pattern, Map<String, Object> indexedObj) {
-    for(String k : indexedObj.keySet()) {
-      if(k.startsWith(pattern)) {
-        return true;
-      }
-    }
-    return false;
-  }
-  private static Set<String> setOf(String... items) {
-    Set<String> ret = new HashSet<>();
-    for(String item : items) {
-      ret.add(item);
-    }
-    return ret;
-  }
-
-  abstract public FieldNameConverter getFieldNameConverter();
-  abstract public InMemoryComponent getSearchComponent(Properties topologyProperties) throws Exception;
-  abstract public Processor<List<Map<String, Object>>> getProcessor(List<byte[]> inputMessages);
-  abstract public void setAdditionalProperties(Properties topologyProperties);
-  abstract public String cleanField(String field);
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
deleted file mode 100644
index 7e0f325..0000000
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.components;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
-import org.apache.metron.common.configuration.SensorParserConfig;
-import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.integration.InMemoryComponent;
-import org.apache.metron.integration.UnableToStartException;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Properties;
-
-public class ConfigUploadComponent implements InMemoryComponent {
-
-  private Properties topologyProperties;
-  private String globalConfigPath;
-  private String parserConfigsPath;
-  private String enrichmentConfigsPath;
-  private Optional<String> globalConfig = Optional.empty();
-  private Map<String, SensorParserConfig> parserSensorConfigs = new HashMap<>();
-  public ConfigUploadComponent withTopologyProperties(Properties topologyProperties) {
-    this.topologyProperties = topologyProperties;
-    return this;
-  }
-
-  public ConfigUploadComponent withGlobalConfigsPath(String globalConfigPath) {
-    this.globalConfigPath = globalConfigPath;
-    return this;
-  }
-
-  public ConfigUploadComponent withParserConfigsPath(String parserConfigsPath) {
-    this.parserConfigsPath = parserConfigsPath;
-    return this;
-  }
-  public ConfigUploadComponent withEnrichmentConfigsPath(String enrichmentConfigsPath) {
-    this.enrichmentConfigsPath = enrichmentConfigsPath;
-    return this;
-  }
-
-  public ConfigUploadComponent withParserSensorConfig(String name, SensorParserConfig config) {
-    parserSensorConfigs.put(name, config);
-    return this;
-  }
-
-  public ConfigUploadComponent withGlobalConfig(String globalConfig) {
-    this.globalConfig = Optional.ofNullable(globalConfig);
-    return this;
-  }
-
-  @Override
-  public void start() throws UnableToStartException {
-    try {
-      if(globalConfigPath != null) {
-        ConfigurationsUtils.uploadConfigsToZookeeper(globalConfigPath
-                , parserConfigsPath
-                , enrichmentConfigsPath
-                , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
-        );
-      }
-      for(Map.Entry<String, SensorParserConfig> kv : parserSensorConfigs.entrySet()) {
-        ConfigurationsUtils.writeSensorParserConfigToZookeeper( kv.getKey()
-                                                              , kv.getValue()
-                                                              , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
-                                                              );
-      }
-      if(globalConfig.isPresent()) {
-        ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig.get().getBytes()
-                                                        , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
-                                                        );
-      }
-
-    } catch (Exception e) {
-      throw new UnableToStartException(e.getMessage(), e);
-    }
-  }
-
-  public SensorParserConfig getSensorParserConfig(String sensorType) {
-    SensorParserConfig sensorParserConfig = new SensorParserConfig();
-    CuratorFramework client = ConfigurationsUtils.getClient(topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY));
-    client.start();
-    try {
-      sensorParserConfig = ConfigurationsUtils.readSensorParserConfigFromZookeeper(sensorType, client);
-    } catch (Exception e) {
-      e.printStackTrace();
-    } finally {
-      client.close();
-    }
-    return sensorParserConfig;
-  }
-
-  @Override
-  public void stop() {
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockGeoAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockGeoAdapter.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockGeoAdapter.java
deleted file mode 100644
index 7316112..0000000
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockGeoAdapter.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.mock;
-
-import com.google.common.base.Joiner;
-import org.apache.metron.enrichment.bolt.CacheKey;
-import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-import org.json.simple.JSONObject;
-
-import java.io.Serializable;
-
-public class MockGeoAdapter implements EnrichmentAdapter<CacheKey>,
-        Serializable {
-
-  public static final String DEFAULT_LOC_ID = "1";
-  public static final String DEFAULT_COUNTRY = "test country";
-  public static final String DEFAULT_CITY = "test city";
-  public static final String DEFAULT_POSTAL_CODE = "test postalCode";
-  public static final String DEFAULT_LATITUDE = "test latitude";
-  public static final String DEFAULT_LONGITUDE = "test longitude";
-  public static final String DEFAULT_DMACODE= "test dmaCode";
-  public static final String DEFAULT_LOCATION_POINT= Joiner.on(',').join(DEFAULT_LATITUDE,DEFAULT_LONGITUDE);
-
-  @Override
-  public void logAccess(CacheKey value) {
-
-  }
-
-  public JSONObject enrich(CacheKey cache ) {
-    JSONObject enriched = new JSONObject();
-    enriched.put("locID", DEFAULT_LOC_ID);
-    enriched.put("country", DEFAULT_COUNTRY);
-    enriched.put("city", DEFAULT_CITY);
-    enriched.put("postalCode", DEFAULT_POSTAL_CODE);
-    enriched.put("latitude", DEFAULT_LATITUDE);
-    enriched.put("longitude", DEFAULT_LONGITUDE);
-    enriched.put("dmaCode", DEFAULT_DMACODE);
-    enriched.put("location_point", DEFAULT_LOCATION_POINT);
-    return enriched;
-  }
-
-  public boolean initializeAdapter() {
-    return true;
-  }
-
-  public void cleanup() {
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockHBaseConnector.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockHBaseConnector.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockHBaseConnector.java
deleted file mode 100644
index f40b366..0000000
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockHBaseConnector.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.mock;
-
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
-import org.apache.metron.hbase.Connector;
-import org.apache.metron.hbase.TupleTableConfig;
-
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-public class MockHBaseConnector extends Connector {
-    static List<Put> puts = Collections.synchronizedList(new ArrayList<Put>());
-    public MockHBaseConnector(TupleTableConfig conf, String _quorum, String _port) throws IOException {
-        super(conf, _quorum, _port);
-    }
-
-    @Override
-    public void put(Put put) throws InterruptedIOException, RetriesExhaustedWithDetailsException {
-        puts.add(put);
-    }
-
-    @Override
-    public void close() {
-
-    }
-    public static void clear() {
-        puts.clear();
-    }
-    public static List<Put> getPuts() {
-        return puts;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java
deleted file mode 100644
index becadce..0000000
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.integration.mock;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.metron.hbase.TableProvider;
-import org.apache.metron.test.mock.MockHTable;
-
-import java.io.IOException;
-import java.io.Serializable;
-
-public class MockTableProvider implements TableProvider, Serializable {
-  static MockHTable.Provider provider = new MockHTable.Provider();
-  @Override
-  public HTableInterface getTable(Configuration config, String tableName) throws IOException {
-    return provider.getTable(config, tableName);
-  }
-  public static void addTable(String tableName, String... cf) {
-    provider.addToCache(tableName, cf);
-  }
-  public static MockHTable getTable(String tableName) {
-    try {
-      return (MockHTable) provider.getTable(null, tableName);
-    } catch (IOException e) {
-      throw new RuntimeException("Unable to get table: " + tableName);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/SampleUtil.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/SampleUtil.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/SampleUtil.java
deleted file mode 100644
index 8e293c0..0000000
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/SampleUtil.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.utils;
-
-import org.apache.metron.TestConstants;
-import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
-import org.apache.metron.common.configuration.EnrichmentConfigurations;
-import org.apache.metron.common.configuration.ParserConfigurations;
-
-import java.io.IOException;
-import java.util.Map;
-
-public class SampleUtil {
-
-  public static Configurations getSampleConfigs() throws IOException {
-    Configurations configurations = new Configurations();
-    configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
-    return configurations;
-  }
-
-  public static ParserConfigurations getSampleParserConfigs() throws IOException {
-    ParserConfigurations configurations = new ParserConfigurations();
-    configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
-    Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH);
-    for(String sensorType: sensorParserConfigs.keySet()) {
-      configurations.updateSensorParserConfig(sensorType, sensorParserConfigs.get(sensorType));
-    }
-    return configurations;
-  }
-
-  public static EnrichmentConfigurations getSampleEnrichmentConfigs() throws IOException {
-    EnrichmentConfigurations configurations = new EnrichmentConfigurations();
-    configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
-    Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.SAMPLE_CONFIG_PATH);
-    for(String sensorType: sensorEnrichmentConfigs.keySet()) {
-      configurations.updateSensorEnrichmentConfig(sensorType, sensorEnrichmentConfigs.get(sensorType));
-    }
-    return configurations;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index c801a7f..47f7bdd 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -33,8 +33,20 @@
         </dependency>
         <dependency>
             <groupId>org.apache.metron</groupId>
+            <artifactId>metron-writer</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-enrichment</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
             <artifactId>metron-enrichment</artifactId>
             <version>${project.parent.version}</version>
+            <scope>test</scope>
+            <type>test-jar</type>
         </dependency>
         <dependency>
             <groupId>org.apache.metron</groupId>
@@ -108,12 +120,12 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.2.3</version>
+            <version>${global_jackson_version}</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>
-            <version>2.2.3</version>
+            <version>${global_jackson_version}</version>
         </dependency>
         <dependency>
             <groupId>io.thekraken</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/asa/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/asa/remote.yaml b/metron-platform/metron-parsers/src/main/flux/asa/remote.yaml
index 052728e..828e1a5 100644
--- a/metron-platform/metron-parsers/src/main/flux/asa/remote.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/asa/remote.yaml
@@ -23,7 +23,7 @@ components:
     -   id: "parser"
         className: "org.apache.metron.parsers.asa.GrokAsaParser"
     -   id: "writer"
-        className: "org.apache.metron.parsers.writer.KafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         constructorArgs:
             - "${kafka.broker}"
     -   id: "zkHosts"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/asa/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/asa/test.yaml b/metron-platform/metron-parsers/src/main/flux/asa/test.yaml
index c816b45..4694fff 100644
--- a/metron-platform/metron-parsers/src/main/flux/asa/test.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/asa/test.yaml
@@ -23,7 +23,7 @@ components:
     -   id: "parser"
         className: "org.apache.metron.parsers.asa.GrokAsaParser"
     -   id: "writer"
-        className: "org.apache.metron.parsers.writer.KafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         constructorArgs:
             - "${kafka.broker}"
     -   id: "zkHosts"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/fireeye/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/fireeye/remote.yaml b/metron-platform/metron-parsers/src/main/flux/fireeye/remote.yaml
index a745d38..dc1ff6b 100644
--- a/metron-platform/metron-parsers/src/main/flux/fireeye/remote.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/fireeye/remote.yaml
@@ -22,7 +22,7 @@ components:
     -   id: "parser"
         className: "org.apache.metron.parsers.fireeye.BasicFireEyeParser"
     -   id: "writer"
-        className: "org.apache.metron.parsers.writer.KafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         constructorArgs:
             - "${kafka.broker}"
     -   id: "zkHosts"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/fireeye/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/fireeye/test.yaml b/metron-platform/metron-parsers/src/main/flux/fireeye/test.yaml
index 9f4c06f..f676b2a 100644
--- a/metron-platform/metron-parsers/src/main/flux/fireeye/test.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/fireeye/test.yaml
@@ -22,7 +22,7 @@ components:
     -   id: "parser"
         className: "org.apache.metron.parsers.fireeye.BasicFireEyeParser"
     -   id: "writer"
-        className: "org.apache.metron.parsers.writer.KafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         constructorArgs:
             - "${kafka.broker}"
     -   id: "zkHosts"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/ise/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/ise/remote.yaml b/metron-platform/metron-parsers/src/main/flux/ise/remote.yaml
index 53ed5fc..49c4c19 100644
--- a/metron-platform/metron-parsers/src/main/flux/ise/remote.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/ise/remote.yaml
@@ -22,7 +22,7 @@ components:
     -   id: "parser"
         className: "org.apache.metron.parsers.ise.BasicIseParser"
     -   id: "writer"
-        className: "org.apache.metron.parsers.writer.KafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         constructorArgs:
             - "${kafka.broker}"
     -   id: "zkHosts"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/ise/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/ise/test.yaml b/metron-platform/metron-parsers/src/main/flux/ise/test.yaml
index 2e88594..8b292e0 100644
--- a/metron-platform/metron-parsers/src/main/flux/ise/test.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/ise/test.yaml
@@ -22,7 +22,7 @@ components:
     -   id: "parser"
         className: "org.apache.metron.parsers.ise.BasicIseParser"
     -   id: "writer"
-        className: "org.apache.metron.parsers.writer.KafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         constructorArgs:
             - "${kafka.broker}"
     -   id: "zkHosts"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/lancope/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/lancope/remote.yaml b/metron-platform/metron-parsers/src/main/flux/lancope/remote.yaml
index 4fe7c3b..f60e86a 100644
--- a/metron-platform/metron-parsers/src/main/flux/lancope/remote.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/lancope/remote.yaml
@@ -22,7 +22,7 @@ components:
     -   id: "parser"
         className: "org.apache.metron.parsers.lancope.BasicLancopeParser"
     -   id: "writer"
-        className: "org.apache.metron.parsers.writer.KafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         constructorArgs:
             - "${kafka.broker}"
     -   id: "zkHosts"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/lancope/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/lancope/test.yaml b/metron-platform/metron-parsers/src/main/flux/lancope/test.yaml
index eb8a1ef..f6ade8e 100644
--- a/metron-platform/metron-parsers/src/main/flux/lancope/test.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/lancope/test.yaml
@@ -22,7 +22,7 @@ components:
     -   id: "parser"
         className: "org.apache.metron.parsers.lancope.BasicLancopeParser"
     -   id: "writer"
-        className: "org.apache.metron.parsers.writer.KafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         constructorArgs:
             - "${kafka.broker}"
     -   id: "zkHosts"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/paloalto/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/paloalto/remote.yaml b/metron-platform/metron-parsers/src/main/flux/paloalto/remote.yaml
index 4287fce..74ea3ad 100644
--- a/metron-platform/metron-parsers/src/main/flux/paloalto/remote.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/paloalto/remote.yaml
@@ -22,7 +22,7 @@ components:
     -   id: "parser"
         className: "org.apache.metron.parsers.paloalto.BasicPaloAltoFirewallParser"
     -   id: "writer"
-        className: "org.apache.metron.parsers.writer.KafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         constructorArgs:
             - "${kafka.broker}"
     -   id: "zkHosts"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/paloalto/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/paloalto/test.yaml b/metron-platform/metron-parsers/src/main/flux/paloalto/test.yaml
index cef5dc5..160b0df 100644
--- a/metron-platform/metron-parsers/src/main/flux/paloalto/test.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/paloalto/test.yaml
@@ -22,7 +22,7 @@ components:
     -   id: "parser"
         className: "org.apache.metron.parsers.paloalto.BasicPaloAltoFirewallParser"
     -   id: "writer"
-        className: "org.apache.metron.parsers.writer.KafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         constructorArgs:
             - "${kafka.broker}"
     -   id: "zkHosts"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/sourcefire/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/sourcefire/remote.yaml b/metron-platform/metron-parsers/src/main/flux/sourcefire/remote.yaml
index 312dce0..a79bfde 100644
--- a/metron-platform/metron-parsers/src/main/flux/sourcefire/remote.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/sourcefire/remote.yaml
@@ -22,7 +22,7 @@ components:
     -   id: "parser"
         className: "org.apache.metron.parsers.sourcefire.BasicSourcefireParser"
     -   id: "writer"
-        className: "org.apache.metron.parsers.writer.KafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         constructorArgs:
             - "${kafka.broker}"
     -   id: "zkHosts"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/sourcefire/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/sourcefire/test.yaml b/metron-platform/metron-parsers/src/main/flux/sourcefire/test.yaml
index a0a00d0..5a6516d 100644
--- a/metron-platform/metron-parsers/src/main/flux/sourcefire/test.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/sourcefire/test.yaml
@@ -22,7 +22,7 @@ components:
     -   id: "parser"
         className: "org.apache.metron.parsers.sourcefire.BasicSourcefireParser"
     -   id: "writer"
-        className: "org.apache.metron.parsers.writer.KafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         constructorArgs:
             - "${kafka.broker}"
     -   id: "zkHosts"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/websphere/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/websphere/remote.yaml b/metron-platform/metron-parsers/src/main/flux/websphere/remote.yaml
index 7d1c103..9f925dd 100644
--- a/metron-platform/metron-parsers/src/main/flux/websphere/remote.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/websphere/remote.yaml
@@ -32,7 +32,7 @@ components:
                 args:
                     - "yyyy MMM dd HH:mm:ss"
     -   id: "writer"
-        className: "org.apache.metron.parsers.writer.KafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         constructorArgs:
             - "${kafka.broker}"
     -   id: "zkHosts"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/websphere/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/websphere/test.yaml b/metron-platform/metron-parsers/src/main/flux/websphere/test.yaml
index 045fc0e..20ac9ca 100644
--- a/metron-platform/metron-parsers/src/main/flux/websphere/test.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/websphere/test.yaml
@@ -32,7 +32,7 @@ components:
                 args:
                     - "yyyy MMM dd HH:mm:ss"
     -   id: "writer"
-        className: "org.apache.metron.parsers.writer.KafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         constructorArgs:
             - "${kafka.broker}"
     -   id: "zkHosts"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
index aebb8d7..6c40768 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
@@ -26,29 +26,19 @@ import backtype.storm.tuple.Values;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.bolt.ConfiguredParserBolt;
 import org.apache.metron.common.configuration.FieldValidator;
-import org.apache.metron.common.configuration.ParserConfigurations;
-import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
-import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFacade;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.interfaces.BulkMessageWriter;
-import org.apache.metron.common.writer.BulkWriterComponent;
-import org.apache.metron.common.writer.WriterToBulkWriter;
 import org.apache.metron.common.configuration.SensorParserConfig;
 import org.apache.metron.parsers.filters.Filters;
 import org.apache.metron.common.configuration.FieldTransformer;
-import org.apache.metron.common.configuration.SensorParserConfig;
 import org.apache.metron.parsers.filters.GenericMessageFilter;
 import org.apache.metron.common.utils.ErrorUtils;
 import org.apache.metron.parsers.interfaces.MessageFilter;
 import org.apache.metron.parsers.interfaces.MessageParser;
-import org.apache.metron.common.interfaces.MessageWriter;
 import org.json.simple.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.Serializable;
 import java.util.*;
-import java.util.function.Function;
 
 public class ParserBolt extends ConfiguredParserBolt implements Serializable {
 



[2/5] incubator-metron git commit: METRON-154: Decouple enrichment and indexing closes apache/incubator-metron#192

Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java
index ecd1ce8..e7ad755 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java
@@ -26,8 +26,8 @@ import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFac
 import org.apache.metron.common.configuration.writer.WriterConfiguration;
 import org.apache.metron.common.interfaces.BulkMessageWriter;
 import org.apache.metron.common.interfaces.MessageWriter;
-import org.apache.metron.common.writer.BulkWriterComponent;
-import org.apache.metron.common.writer.WriterToBulkWriter;
+import org.apache.metron.writer.BulkWriterComponent;
+import org.apache.metron.writer.WriterToBulkWriter;
 import org.json.simple.JSONObject;
 
 import java.io.Serializable;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
index d9004d1..000b7ff 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
@@ -29,12 +29,12 @@ import org.apache.metron.common.interfaces.MessageWriter;
 import org.apache.metron.common.spout.kafka.SpoutConfig;
 import org.apache.metron.common.spout.kafka.SpoutConfigOptions;
 import org.apache.metron.common.utils.ReflectionUtils;
-import org.apache.metron.common.writer.AbstractWriter;
 import org.apache.metron.parsers.bolt.ParserBolt;
 import org.apache.metron.parsers.bolt.WriterBolt;
 import org.apache.metron.parsers.bolt.WriterHandler;
 import org.apache.metron.parsers.interfaces.MessageParser;
-import org.apache.metron.parsers.writer.KafkaWriter;
+import org.apache.metron.writer.AbstractWriter;
+import org.apache.metron.writer.kafka.KafkaWriter;
 import org.json.simple.JSONObject;
 import storm.kafka.KafkaSpout;
 import storm.kafka.ZkHosts;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
deleted file mode 100644
index 6090491..0000000
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers.writer;
-
-import backtype.storm.tuple.Tuple;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.interfaces.MessageWriter;
-import org.apache.metron.common.utils.ConversionUtils;
-import org.apache.metron.common.utils.StringUtils;
-import org.apache.metron.common.writer.AbstractWriter;
-import org.json.simple.JSONObject;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-
-public class KafkaWriter extends AbstractWriter implements MessageWriter<JSONObject>, Serializable {
-  public enum Configurations {
-     BROKER("kafka.brokerUrl")
-    ,KEY_SERIALIZER("kafka.keySerializer")
-    ,VALUE_SERIALIZER("kafka.valueSerializer")
-    ,REQUIRED_ACKS("kafka.requiredAcks")
-    ,TOPIC("kafka.topic")
-    ,PRODUCER_CONFIGS("kafka.producerConfigs");
-    ;
-    String key;
-    Configurations(String key) {
-      this.key = key;
-    }
-    public Object get(Optional<String> configPrefix, Map<String, Object> config) {
-      return config.get(StringUtils.join(".", configPrefix, Optional.of(key)));
-    }
-    public <T> T getAndConvert(Optional<String> configPrefix, Map<String, Object> config, Class<T> clazz) {
-      Object o = get(configPrefix, config);
-      if(o != null) {
-        return ConversionUtils.convert(o, clazz);
-      }
-      return null;
-    }
-  }
-  private String brokerUrl;
-  private String keySerializer = "org.apache.kafka.common.serialization.StringSerializer";
-  private String valueSerializer = "org.apache.kafka.common.serialization.StringSerializer";
-  private int requiredAcks = 1;
-  private String kafkaTopic = Constants.ENRICHMENT_TOPIC;
-  private KafkaProducer kafkaProducer;
-  private String configPrefix = null;
-  private Map<String, Object> producerConfigs = new HashMap<>();
-
-  public KafkaWriter() {}
-
-  public KafkaWriter(String brokerUrl) {
-    this.brokerUrl = brokerUrl;
-  }
-
-  public KafkaWriter withKeySerializer(String keySerializer) {
-    this.keySerializer = keySerializer;
-    return this;
-  }
-
-  public KafkaWriter withValueSerializer(String valueSerializer) {
-    this.valueSerializer = valueSerializer;
-    return this;
-  }
-
-  public KafkaWriter withRequiredAcks(Integer requiredAcks) {
-    this.requiredAcks = requiredAcks;
-    return this;
-  }
-
-  public KafkaWriter withTopic(String topic) {
-    this.kafkaTopic= topic;
-    return this;
-  }
-  public KafkaWriter withConfigPrefix(String prefix) {
-    this.configPrefix = prefix;
-    return this;
-  }
-
-  public KafkaWriter withProducerConfigs(Map<String, Object> extraConfigs) {
-    this.producerConfigs = extraConfigs;
-    return this;
-  }
-
-  public Optional<String> getConfigPrefix() {
-    return Optional.ofNullable(configPrefix);
-  }
-
-  @Override
-  public void configure(String sensorName, WriterConfiguration configuration) {
-    Map<String, Object> configMap = configuration.getSensorConfig(sensorName);
-    String brokerUrl = Configurations.BROKER.getAndConvert(getConfigPrefix(), configMap, String.class);
-    if(brokerUrl != null) {
-      this.brokerUrl = brokerUrl;
-    }
-    String keySerializer = Configurations.KEY_SERIALIZER.getAndConvert(getConfigPrefix(), configMap, String.class);
-    if(keySerializer != null) {
-      withKeySerializer(keySerializer);
-    }
-    String valueSerializer = Configurations.VALUE_SERIALIZER.getAndConvert(getConfigPrefix(), configMap, String.class);
-    if(valueSerializer != null) {
-      withValueSerializer(keySerializer);
-    }
-    Integer requiredAcks = Configurations.REQUIRED_ACKS.getAndConvert(getConfigPrefix(), configMap, Integer.class);
-    if(requiredAcks!= null) {
-      withRequiredAcks(requiredAcks);
-    }
-    String topic = Configurations.TOPIC.getAndConvert(getConfigPrefix(), configMap, String.class);
-    if(topic != null) {
-      withTopic(topic);
-    }
-    Map<String, Object> producerConfigs = (Map)Configurations.PRODUCER_CONFIGS.get(getConfigPrefix(), configMap);
-    if(producerConfigs != null) {
-      withProducerConfigs(producerConfigs);
-    }
-  }
-
-  public Map<String, Object> createProducerConfigs() {
-    Map<String, Object> producerConfig = new HashMap<>();
-    producerConfig.put("bootstrap.servers", brokerUrl);
-    producerConfig.put("key.serializer", keySerializer);
-    producerConfig.put("value.serializer", valueSerializer);
-    producerConfig.put("request.required.acks", requiredAcks);
-    producerConfig.putAll(producerConfigs == null?new HashMap<>():producerConfigs);
-    return producerConfig;
-  }
-
-  @Override
-  public void init() {
-
-    this.kafkaProducer = new KafkaProducer<>(createProducerConfigs());
-  }
-
-  @SuppressWarnings("unchecked")
-  @Override
-  public void write(String sourceType, WriterConfiguration configurations, Tuple tuple, JSONObject message) throws Exception {
-    kafkaProducer.send(new ProducerRecord<String, String>(kafkaTopic, message.toJSONString()));
-  }
-
-  @Override
-  public void close() throws Exception {
-    kafkaProducer.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
index 116e262..b3d09d0 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
@@ -18,25 +18,21 @@
 package org.apache.metron.parsers.integration;
 
 import junit.framework.Assert;
-import org.apache.commons.io.FilenameUtils;
 import org.apache.metron.TestConstants;
 import org.apache.metron.common.Constants;
+import org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
 import org.apache.metron.integration.BaseIntegrationTest;
 import org.apache.metron.integration.ComponentRunner;
 import org.apache.metron.integration.Processor;
 import org.apache.metron.integration.ReadinessState;
-import org.apache.metron.integration.components.ConfigUploadComponent;
 import org.apache.metron.integration.components.KafkaWithZKComponent;
 import org.apache.metron.integration.utils.TestUtils;
 import org.apache.metron.parsers.integration.components.ParserTopologyComponent;
-import org.apache.metron.parsers.integration.validation.SampleDataValidation;
 import org.apache.metron.test.TestDataType;
 import org.apache.metron.test.utils.SampleDataUtils;
 import org.apache.metron.test.utils.UnitTestHelper;
-import org.junit.Before;
 import org.junit.Test;
 
-import java.io.File;
 import java.util.*;
 
 public abstract class ParserIntegrationTest extends BaseIntegrationTest {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/writer/KafkaWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/writer/KafkaWriterTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/writer/KafkaWriterTest.java
deleted file mode 100644
index 57dc3e2..0000000
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/writer/KafkaWriterTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.parsers.writer;
-
-import com.google.common.collect.ImmutableMap;
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.test.TestingServer;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
-import org.apache.metron.common.configuration.ParserConfigurations;
-import org.apache.metron.common.configuration.SensorParserConfig;
-import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-public class KafkaWriterTest {
-
-
-  public static final String SENSOR_TYPE = "test";
-  public WriterConfiguration createConfiguration(final Map<String, Object> parserConfig) {
-    ParserConfigurations configurations = new ParserConfigurations();
-    configurations.updateSensorParserConfig( SENSOR_TYPE
-                                           , new SensorParserConfig() {{
-                                              setParserConfig(parserConfig);
-                                                                      }}
-                                           );
-    return new ParserWriterConfiguration(configurations);
-  }
-
-  @Test
-  public void testHappyPathGlobalConfig() throws Exception {
-    KafkaWriter writer = new KafkaWriter();
-    WriterConfiguration configuration = createConfiguration(
-            new HashMap<String, Object>() {{
-              put("kafka.brokerUrl" , "localhost:6667");
-              put("kafka.topic" , SENSOR_TYPE);
-              put("kafka.producerConfigs" , ImmutableMap.of("key1", 1, "key2", "value2"));
-            }}
-    );
-
-    writer.configure(SENSOR_TYPE, configuration);
-    Map<String, Object> producerConfigs = writer.createProducerConfigs();
-    Assert.assertEquals(producerConfigs.get("bootstrap.servers"), "localhost:6667");
-    Assert.assertEquals(producerConfigs.get("key.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
-    Assert.assertEquals(producerConfigs.get("value.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
-    Assert.assertEquals(producerConfigs.get("request.required.acks"), 1);
-    Assert.assertEquals(producerConfigs.get("key1"), 1);
-    Assert.assertEquals(producerConfigs.get("key2"), "value2");
-  }
-
-  @Test
-  public void testHappyPathGlobalConfigWithPrefix() throws Exception {
-    KafkaWriter writer = new KafkaWriter();
-    writer.withConfigPrefix("prefix");
-    WriterConfiguration configuration = createConfiguration(
-            new HashMap<String, Object>() {{
-              put("prefix.kafka.brokerUrl" , "localhost:6667");
-              put("prefix.kafka.topic" , SENSOR_TYPE);
-              put("prefix.kafka.producerConfigs" , ImmutableMap.of("key1", 1, "key2", "value2"));
-            }}
-    );
-
-    writer.configure(SENSOR_TYPE, configuration);
-    Map<String, Object> producerConfigs = writer.createProducerConfigs();
-    Assert.assertEquals(producerConfigs.get("bootstrap.servers"), "localhost:6667");
-    Assert.assertEquals(producerConfigs.get("key.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
-    Assert.assertEquals(producerConfigs.get("value.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
-    Assert.assertEquals(producerConfigs.get("request.required.acks"), 1);
-    Assert.assertEquals(producerConfigs.get("key1"), 1);
-    Assert.assertEquals(producerConfigs.get("key2"), "value2");
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
index cea635f..a21d45b 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
@@ -20,17 +20,16 @@ package org.apache.metron.writers;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.metron.common.configuration.writer.WriterConfiguration;
 import org.apache.metron.enrichment.converter.EnrichmentConverter;
 import org.apache.metron.enrichment.converter.EnrichmentKey;
 import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.integration.mock.MockTableProvider;
 import org.apache.metron.enrichment.lookup.LookupKV;
-import org.apache.metron.integration.mock.MockTableProvider;
 import org.apache.metron.test.mock.MockHTable;
-import org.apache.metron.writer.hbase.SimpleHbaseEnrichmentWriter;
+import org.apache.metron.enrichment.writer.SimpleHbaseEnrichmentWriter;
 import org.json.simple.JSONObject;
 import org.junit.Assert;
 import org.junit.Before;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
index f6436b9..2f359ef 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
@@ -20,31 +20,25 @@ package org.apache.metron.writers.integration;
 
 import com.google.common.collect.ImmutableList;
 import org.adrianwalker.multilinestring.Multiline;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.metron.TestConstants;
-import org.apache.metron.common.Constants;
 import org.apache.metron.common.configuration.SensorParserConfig;
 import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.enrichment.converter.EnrichmentConverter;
 import org.apache.metron.enrichment.converter.EnrichmentKey;
 import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
+import org.apache.metron.enrichment.integration.mock.MockTableProvider;
 import org.apache.metron.enrichment.lookup.LookupKV;
 import org.apache.metron.integration.*;
-import org.apache.metron.integration.components.ConfigUploadComponent;
 import org.apache.metron.integration.components.KafkaWithZKComponent;
-import org.apache.metron.integration.mock.MockTableProvider;
-import org.apache.metron.integration.utils.TestUtils;
 import org.apache.metron.parsers.integration.components.ParserTopologyComponent;
-import org.apache.metron.test.TestDataType;
 import org.apache.metron.test.mock.MockHTable;
-import org.apache.metron.test.utils.SampleDataUtils;
 import org.apache.metron.test.utils.UnitTestHelper;
 import org.junit.Assert;
 import org.junit.Test;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.*;
 
@@ -53,7 +47,7 @@ public class SimpleHbaseEnrichmentWriterIntegrationTest extends BaseIntegrationT
   /**
    {
     "parserClassName" : "org.apache.metron.parsers.csv.CSVParser"
-   ,"writerClassName" : "org.apache.metron.writer.hbase.SimpleHbaseEnrichmentWriter"
+   ,"writerClassName" : "org.apache.metron.enrichment.writer.SimpleHbaseEnrichmentWriter"
    ,"sensorTopic":"dummy"
    ,"parserConfig":
    {
@@ -61,7 +55,7 @@ public class SimpleHbaseEnrichmentWriterIntegrationTest extends BaseIntegrationT
     ,"shew.cf" : "cf"
     ,"shew.keyColumns" : "col2"
     ,"shew.enrichmentType" : "et"
-    ,"shew.hbaseProvider" : "org.apache.metron.integration.mock.MockTableProvider"
+    ,"shew.hbaseProvider" : "org.apache.metron.enrichment.integration.mock.MockTableProvider"
     ,"columns" : {
                 "col1" : 0
                ,"col2" : 1

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
index a6e83f1..afdcf04 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
@@ -18,37 +18,22 @@
 
 package org.apache.metron.writers.integration;
 import com.fasterxml.jackson.core.type.TypeReference;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import org.adrianwalker.multilinestring.Multiline;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.metron.TestConstants;
 import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.FieldValidator;
 import org.apache.metron.common.configuration.SensorParserConfig;
 import org.apache.metron.common.field.validation.FieldValidation;
 import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.enrichment.converter.EnrichmentConverter;
-import org.apache.metron.enrichment.converter.EnrichmentKey;
-import org.apache.metron.enrichment.converter.EnrichmentValue;
-import org.apache.metron.enrichment.lookup.LookupKV;
+import org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
 import org.apache.metron.integration.*;
-import org.apache.metron.integration.components.ConfigUploadComponent;
 import org.apache.metron.integration.components.KafkaWithZKComponent;
-import org.apache.metron.integration.mock.MockTableProvider;
-import org.apache.metron.integration.utils.TestUtils;
 import org.apache.metron.parsers.integration.components.ParserTopologyComponent;
-import org.apache.metron.test.TestDataType;
-import org.apache.metron.test.mock.MockHTable;
-import org.apache.metron.test.utils.SampleDataUtils;
 import org.apache.metron.test.utils.UnitTestHelper;
 import org.json.simple.JSONObject;
 import org.junit.Assert;
 import org.junit.Test;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.*;
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-solr/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml
index 54b50f4..3aceda0 100644
--- a/metron-platform/metron-solr/pom.xml
+++ b/metron-platform/metron-solr/pom.xml
@@ -32,16 +32,16 @@
             <version>${global_hbase_guava_version}</version>
         </dependency>
         <dependency>
-            <groupId>org.apache.metron</groupId>
-            <artifactId>metron-enrichment</artifactId>
-            <version>${project.parent.version}</version>
-        </dependency>
-        <dependency>
             <groupId>org.apache.solr</groupId>
             <artifactId>solr-solrj</artifactId>
             <version>${global_solr_version}</version>
         </dependency>
         <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-enrichment</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
             <groupId>org.apache.solr</groupId>
             <artifactId>solr-test-framework</artifactId>
             <version>${global_solr_version}</version>
@@ -98,6 +98,10 @@
                     <artifactId>servlet-api</artifactId>
                     <groupId>javax.servlet</groupId>
                 </exclusion>
+                <exclusion>
+                    <artifactId>commons-httpclient</artifactId>
+                    <groupId>commons-httpclient</groupId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -122,8 +126,8 @@
                     <groupId>javax.servlet</groupId>
                 </exclusion>
                 <exclusion>
-                    <groupId>org.apache.httpcomponents</groupId>
-                    <artifactId>httpclient</artifactId>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-common</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
@@ -133,6 +137,37 @@
             <version>${global_mockito_version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-indexing</artifactId>
+            <version>${project.parent.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-enrichment</artifactId>
+            <version>${project.parent.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-common</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-indexing</artifactId>
+            <version>${project.parent.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.httpcomponents</groupId>
+                    <artifactId>httpclient</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
 
     </dependencies>
 
@@ -201,6 +236,7 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+
                             <artifactSet>
                                 <excludes>
                                     <exclude>storm:storm-core:*</exclude>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-solr/src/main/config/solr.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/solr.properties b/metron-platform/metron-solr/src/main/config/solr.properties
index cdfe25a..8489d8e 100644
--- a/metron-platform/metron-solr/src/main/config/solr.properties
+++ b/metron-platform/metron-solr/src/main/config/solr.properties
@@ -14,37 +14,21 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+##### Storm #####
+indexing.workers=1
+indexing.executors=0
 
 ##### Kafka #####
 
 kafka.zk=node1:2181
 kafka.broker=node1:6667
-spout.kafka.topic.asa=asa
-spout.kafka.topic.bro=bro
-spout.kafka.topic.fireeye=fireeye
-spout.kafka.topic.ise=ise
-spout.kafka.topic.lancope=lancope
-spout.kafka.topic.paloalto=paloalto
-spout.kafka.topic.pcap=pcap
-spout.kafka.topic.snort=snort
-spout.kafka.topic.yaf=yaf
+kafka.start=WHERE_I_LEFT_OFF
 
 ##### Indexing #####
+index.input.topic=indexing
+index.error.topic=indexing_error
 writer.class.name=org.apache.metron.solr.writer.SolrWriter
 
-##### ElasticSearch #####
-
-es.ip=10.22.0.214
-es.port=9300
-es.clustername=elasticsearch
-
-##### MySQL #####
-
-mysql.ip=10.22.0.214
-mysql.port=3306
-mysql.username=root
-mysql.password=hadoop123
-
 ##### Metrics #####
 
 #reporters
@@ -63,23 +47,6 @@ org.apache.metron.metrics.TelemetryParserBolt.emits=true
 org.apache.metron.metrics.TelemetryParserBolt.fails=true
 
 
-#GenericEnrichmentBolt
-org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
-org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
-org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
-
-
-#TelemetryIndexingBolt
-org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
-org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
-org.apache.metron.metrics.TelemetryIndexingBolt.fails=true
-
-##### Host Enrichment #####
-
-org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
-{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
-{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
-
 ##### HDFS #####
 
 bolt.hdfs.batch.size=5000
@@ -91,19 +58,3 @@ bolt.hdfs.finished.file.path=/paloalto/rotated
 bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
 index.hdfs.output=/tmp/metron/enriched
 
-##### HBase #####
-bolt.hbase.table.name=pcap
-bolt.hbase.table.fields=t:value
-bolt.hbase.table.key.tuple.field.name=key
-bolt.hbase.table.timestamp.tuple.field.name=timestamp
-bolt.hbase.enable.batching=false
-bolt.hbase.write.buffer.size.in.bytes=2000000
-bolt.hbase.durability=SKIP_WAL
-bolt.hbase.partitioner.region.info.refresh.interval.mins=60
-
-##### Threat Intel #####
-
-threat.intel.tracker.table=
-threat.intel.tracker.cf=
-threat.intel.ip.table=
-threat.intel.ip.cf=

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-solr/src/main/scripts/start_solr_topology.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/scripts/start_solr_topology.sh b/metron-platform/metron-solr/src/main/scripts/start_solr_topology.sh
index 14628d2..7a98fc7 100755
--- a/metron-platform/metron-solr/src/main/scripts/start_solr_topology.sh
+++ b/metron-platform/metron-solr/src/main/scripts/start_solr_topology.sh
@@ -19,4 +19,4 @@
 METRON_VERSION=${project.version}
 METRON_HOME=/usr/metron/$METRON_VERSION
 TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION.jar
-storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/enrichment/remote.yaml --filter $METRON_HOME/config/solr.properties
+storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/remote.yaml --filter $METRON_HOME/config/solr.properties

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrEnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrEnrichmentIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrEnrichmentIntegrationTest.java
deleted file mode 100644
index 31e8d49..0000000
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrEnrichmentIntegrationTest.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.solr.integration;
-
-import com.google.common.base.Function;
-import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.common.interfaces.FieldNameConverter;
-import org.apache.metron.integration.EnrichmentIntegrationTest;
-import org.apache.metron.integration.ComponentRunner;
-import org.apache.metron.integration.InMemoryComponent;
-import org.apache.metron.integration.Processor;
-import org.apache.metron.integration.ReadinessState;
-import org.apache.metron.integration.components.KafkaWithZKComponent;
-import org.apache.metron.solr.integration.components.SolrComponent;
-import org.apache.metron.integration.utils.SampleUtil;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
-import org.apache.metron.common.utils.JSONUtils;
-
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-public class SolrEnrichmentIntegrationTest extends EnrichmentIntegrationTest {
-
-  private String collection = "metron";
-  private FieldNameConverter fieldNameConverter = fieldName -> fieldName;
-  @Override
-  public FieldNameConverter getFieldNameConverter() {
-    return fieldNameConverter;
-  }
-
-  @Override
-  public InMemoryComponent getSearchComponent(final Properties topologyProperties) throws Exception {
-    SolrComponent solrComponent = new SolrComponent.Builder()
-            .addCollection(collection, "../metron-solr/src/test/resources/solr/conf")
-            .withPostStartCallback(new Function<SolrComponent, Void>() {
-              @Nullable
-              @Override
-              public Void apply(@Nullable SolrComponent solrComponent) {
-                topologyProperties.setProperty("solr.zk", solrComponent.getZookeeperUrl());
-                try {
-                  String testZookeeperUrl = topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY);
-                  Configurations configurations = SampleUtil.getSampleConfigs();
-                  Map<String, Object> globalConfig = configurations.getGlobalConfig();
-                  globalConfig.put("solr.zookeeper", solrComponent.getZookeeperUrl());
-                  ConfigurationsUtils.writeGlobalConfigToZookeeper(JSONUtils.INSTANCE.toJSON(globalConfig), testZookeeperUrl);
-                } catch (Exception e) {
-                  e.printStackTrace();
-                }
-                return null;
-              }
-            })
-            .build();
-    return solrComponent;
-  }
-
-  @Override
-  public Processor<List<Map<String, Object>>> getProcessor(final List<byte[]> inputMessages) {
-    return new Processor<List<Map<String, Object>>>() {
-      List<Map<String, Object>> docs = null;
-      public ReadinessState process(ComponentRunner runner) {
-        SolrComponent solrComponent = runner.getComponent("search", SolrComponent.class);
-        if (solrComponent.hasCollection(collection)) {
-          List<Map<String, Object>> docsFromDisk;
-          try {
-            docs = solrComponent.getAllIndexedDocs(collection);
-            docsFromDisk = EnrichmentIntegrationTest.readDocsFromDisk(hdfsDir);
-            System.out.println(docs.size() + " vs " + inputMessages.size() + " vs " + docsFromDisk.size());
-          } catch (IOException e) {
-            throw new IllegalStateException("Unable to retrieve indexed documents.", e);
-          }
-          if (docs.size() < inputMessages.size() || docs.size() != docsFromDisk.size()) {
-            return ReadinessState.NOT_READY;
-          } else {
-            return ReadinessState.READY;
-          }
-        } else {
-          return ReadinessState.NOT_READY;
-        }
-      }
-
-      public List<Map<String, Object>> getResult() {
-        return docs;
-      }
-    };
-  }
-
-  @Override
-  public void setAdditionalProperties(Properties topologyProperties) {
-    topologyProperties.setProperty("writer.class.name", "org.apache.metron.solr.writer.SolrWriter");
-  }
-
-  @Override
-  public String cleanField(String field) {
-    return field.replaceFirst("_[dfils]$", "");
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
new file mode 100644
index 0000000..dd9c559
--- /dev/null
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.solr.integration;
+
+import com.google.common.base.Function;
+import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.interfaces.FieldNameConverter;
+import org.apache.metron.enrichment.integration.utils.SampleUtil;
+import org.apache.metron.indexing.integration.IndexingIntegrationTest;
+import org.apache.metron.integration.ComponentRunner;
+import org.apache.metron.integration.InMemoryComponent;
+import org.apache.metron.integration.Processor;
+import org.apache.metron.integration.ReadinessState;
+import org.apache.metron.integration.components.KafkaWithZKComponent;
+import org.apache.metron.solr.integration.components.SolrComponent;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.utils.JSONUtils;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class SolrIndexingIntegrationTest extends IndexingIntegrationTest {
+
+  private String collection = "metron";
+  private FieldNameConverter fieldNameConverter = fieldName -> fieldName;
+  @Override
+  public FieldNameConverter getFieldNameConverter() {
+    return fieldNameConverter;
+  }
+
+  @Override
+  public InMemoryComponent getSearchComponent(final Properties topologyProperties) throws Exception {
+    SolrComponent solrComponent = new SolrComponent.Builder()
+            .addCollection(collection, "../metron-solr/src/test/resources/solr/conf")
+            .withPostStartCallback(new Function<SolrComponent, Void>() {
+              @Nullable
+              @Override
+              public Void apply(@Nullable SolrComponent solrComponent) {
+                topologyProperties.setProperty("solr.zk", solrComponent.getZookeeperUrl());
+                try {
+                  String testZookeeperUrl = topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY);
+                  Configurations configurations = SampleUtil.getSampleConfigs();
+                  Map<String, Object> globalConfig = configurations.getGlobalConfig();
+                  globalConfig.put("solr.zookeeper", solrComponent.getZookeeperUrl());
+                  ConfigurationsUtils.writeGlobalConfigToZookeeper(JSONUtils.INSTANCE.toJSON(globalConfig), testZookeeperUrl);
+                } catch (Exception e) {
+                  e.printStackTrace();
+                }
+                return null;
+              }
+            })
+            .build();
+    return solrComponent;
+  }
+
+  @Override
+  public Processor<List<Map<String, Object>>> getProcessor(final List<byte[]> inputMessages) {
+    return new Processor<List<Map<String, Object>>>() {
+      List<Map<String, Object>> docs = null;
+      public ReadinessState process(ComponentRunner runner) {
+        SolrComponent solrComponent = runner.getComponent("search", SolrComponent.class);
+        if (solrComponent.hasCollection(collection)) {
+          List<Map<String, Object>> docsFromDisk;
+          try {
+            docs = solrComponent.getAllIndexedDocs(collection);
+            docsFromDisk = readDocsFromDisk(hdfsDir);
+            System.out.println(docs.size() + " vs " + inputMessages.size() + " vs " + docsFromDisk.size());
+          } catch (IOException e) {
+            throw new IllegalStateException("Unable to retrieve indexed documents.", e);
+          }
+          if (docs.size() < inputMessages.size() || docs.size() != docsFromDisk.size()) {
+            return ReadinessState.NOT_READY;
+          } else {
+            return ReadinessState.READY;
+          }
+        } else {
+          return ReadinessState.NOT_READY;
+        }
+      }
+
+      public List<Map<String, Object>> getResult() {
+        return docs;
+      }
+    };
+  }
+
+  @Override
+  public void setAdditionalProperties(Properties topologyProperties) {
+    topologyProperties.setProperty("writer.class.name", "org.apache.metron.solr.writer.SolrWriter");
+  }
+
+  @Override
+  public String cleanField(String field) {
+    return field.replaceFirst("_[dfils]$", "");
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
index 0993e0d..91c34af 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
@@ -17,11 +17,9 @@
  */
 package org.apache.metron.solr.writer;
 
-import backtype.storm.tuple.Tuple;
-import org.apache.metron.common.configuration.Configurations;
 import org.apache.metron.common.configuration.EnrichmentConfigurations;
 import org.apache.metron.common.configuration.writer.EnrichmentWriterConfiguration;
-import org.apache.metron.integration.utils.SampleUtil;
+import org.apache.metron.enrichment.integration.utils.SampleUtil;
 import org.apache.solr.client.solrj.request.QueryRequest;
 import org.apache.solr.common.SolrInputDocument;
 import org.hamcrest.Description;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml
new file mode 100644
index 0000000..d088c4b
--- /dev/null
+++ b/metron-platform/metron-writer/pom.xml
@@ -0,0 +1,231 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software
+	Foundation (ASF) under one or more contributor license agreements. See the
+	NOTICE file distributed with this work for additional information regarding
+	copyright ownership. The ASF licenses this file to You under the Apache License,
+	Version 2.0 (the "License"); you may not use this file except in compliance
+	with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+	Unless required by applicable law or agreed to in writing, software distributed
+	under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+	OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+  the specific language governing permissions and limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.metron</groupId>
+        <artifactId>metron-platform</artifactId>
+        <version>0.2.0BETA</version>
+    </parent>
+    <artifactId>metron-writer</artifactId>
+    <name>metron-writer</name>
+    <description>Components common to all enrichments</description>
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <commons.config.version>1.10</commons.config.version>
+        <antlr.version>4.5</antlr.version>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-common</artifactId>
+            <version>${global_hbase_version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-client</artifactId>
+            <version>${global_hbase_version}</version>
+            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.storm</groupId>
+            <artifactId>storm-core</artifactId>
+            <version>${global_storm_version}</version>
+            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <artifactId>servlet-api</artifactId>
+                    <groupId>javax.servlet</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>log4j-over-slf4j</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka_2.9.2</artifactId>
+            <version>${global_kafka_version}</version>
+            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.sun.jmx</groupId>
+                    <artifactId>jmxri</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jdmk</groupId>
+                    <artifactId>jmxtools</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.jms</groupId>
+                    <artifactId>jms</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.storm</groupId>
+            <artifactId>storm-hdfs</artifactId>
+            <version>${global_storm_version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.storm</groupId>
+                    <artifactId>storm-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-client</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-hdfs</artifactId>
+            <version>${global_hadoop_version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>servlet-api</artifactId>
+                    <groupId>javax.servlet</groupId>
+                </exclusion>
+            </exclusions>
+            <scope>provided</scope>
+        </dependency>
+         <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-common</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+    </dependencies>
+
+    <reporting>
+        <plugins>
+            <!-- Normally, dependency report takes time, skip it -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-project-info-reports-plugin</artifactId>
+                <version>2.7</version>
+
+                <configuration>
+                    <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>emma-maven-plugin</artifactId>
+                <version>1.0-alpha-3</version>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-pmd-plugin</artifactId>
+                <configuration>
+                  <targetJdk>${global_java_version}</targetJdk>
+                </configuration>
+            </plugin>
+        </plugins>
+    </reporting>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.1</version>
+                <configuration>
+                    <source>${global_java_version}</source>
+                    <compilerArgument>-Xlint:unchecked</compilerArgument>
+                    <target>${global_java_version}</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.antlr</groupId>
+                <artifactId>antlr4-maven-plugin</artifactId>
+                <version>${antlr.version}</version>
+                <configuration>
+                  <outputDirectory>${basedir}/src/main/java</outputDirectory>
+                </configuration>
+                <executions>
+                  <execution>
+                    <goals>
+                      <goal>antlr4</goal>
+                    </goals>
+                  </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>${global_shade_version}</version>
+                <configuration>
+                    <createDependencyReducedPom>true</createDependencyReducedPom>
+                    <artifactSet>
+                        <excludes>
+                            <exclude>*slf4j*</exclude>
+                        </excludes>
+                    </artifactSet>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <relocations>
+                                <relocation>
+                                    <pattern>com.google.common</pattern>
+                                    <shadedPattern>org.apache.metron.guava</shadedPattern>
+                                </relocation>
+                            </relocations>
+                            <transformers>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+                                    <resource>.yaml</resource>
+                                </transformer>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass></mainClass>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+            </resource>
+        </resources>
+    </build>
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/AbstractWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/AbstractWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/AbstractWriter.java
new file mode 100644
index 0000000..0fa596a
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/AbstractWriter.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer;
+
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+
+public abstract class AbstractWriter {
+  public AbstractWriter() {}
+  public abstract void configure(String sensorName, WriterConfiguration configuration);
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
new file mode 100644
index 0000000..ba647bd
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.tuple.Tuple;
+import com.google.common.collect.Iterables;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.utils.ErrorUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+public class BulkWriterComponent<MESSAGE_T> {
+  public static final Logger LOG = LoggerFactory
+            .getLogger(BulkWriterComponent.class);
+  private Map<String, Collection<Tuple>> sensorTupleMap = new HashMap<>();
+  private Map<String, List<MESSAGE_T>> sensorMessageMap = new HashMap<>();
+  private OutputCollector collector;
+  private boolean handleCommit = true;
+  private boolean handleError = true;
+  public BulkWriterComponent(OutputCollector collector) {
+    this.collector = collector;
+  }
+
+  public BulkWriterComponent(OutputCollector collector, boolean handleCommit, boolean handleError) {
+    this(collector);
+    this.handleCommit = handleCommit;
+    this.handleError = handleError;
+  }
+
+  public void commit(Iterable<Tuple> tuples) {
+    tuples.forEach(t -> collector.ack(t));
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("Acking " + Iterables.size(tuples) + " tuples");
+    }
+  }
+
+  public void error(Throwable e, Iterable<Tuple> tuples) {
+    tuples.forEach(t -> collector.ack(t));
+    if(!Iterables.isEmpty(tuples)) {
+      LOG.error("Failing " + Iterables.size(tuples) + " tuples", e);
+      ErrorUtils.handleError(collector, e, Constants.ERROR_STREAM);
+    }
+  }
+
+  protected Collection<Tuple> createTupleCollection() {
+    return new ArrayList<>();
+  }
+
+
+  public void errorAll(Throwable e) {
+    for(Map.Entry<String, Collection<Tuple>> kv : sensorTupleMap.entrySet()) {
+      error(e, kv.getValue());
+      sensorTupleMap.remove(kv.getKey());
+      sensorMessageMap.remove(kv.getKey());
+    }
+  }
+
+  public void errorAll(String sensorType, Throwable e) {
+    error(e, Optional.ofNullable(sensorTupleMap.get(sensorType)).orElse(new ArrayList<>()));
+    sensorTupleMap.remove(sensorType);
+    sensorMessageMap.remove(sensorType);
+  }
+  public void write( String sensorType
+                   , Tuple tuple
+                   , MESSAGE_T message
+                   , BulkMessageWriter<MESSAGE_T> bulkMessageWriter
+                   , WriterConfiguration configurations
+                   ) throws Exception
+  {
+    int batchSize = configurations.getBatchSize(sensorType);
+    Collection<Tuple> tupleList = sensorTupleMap.get(sensorType);
+    if (tupleList == null) {
+      tupleList = createTupleCollection();
+    }
+    tupleList.add(tuple);
+    List<MESSAGE_T> messageList = sensorMessageMap.get(sensorType);
+    if (messageList == null) {
+      messageList = new ArrayList<>();
+    }
+    messageList.add(message);
+
+    if (tupleList.size() < batchSize) {
+      sensorTupleMap.put(sensorType, tupleList);
+      sensorMessageMap.put(sensorType, messageList);
+    } else {
+      try {
+        bulkMessageWriter.write(sensorType, configurations, tupleList, messageList);
+        if(handleCommit) {
+          commit(tupleList);
+        }
+
+      } catch (Throwable e) {
+        if(handleError) {
+          error(e, tupleList);
+        }
+        else {
+          throw e;
+        }
+      }
+      finally {
+        sensorTupleMap.remove(sensorType);
+        sensorMessageMap.remove(sensorType);
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java
new file mode 100644
index 0000000..b9aee7e
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer;
+
+import backtype.storm.tuple.Tuple;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.utils.ConversionUtils;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.function.Function;
+
+public class NoopWriter extends AbstractWriter implements BulkMessageWriter<Object> {
+
+  public static class RandomLatency implements Function<Void, Void> {
+    private int min;
+    private int max;
+
+    public RandomLatency(int min, int max) {
+      this.min = min;
+      this.max = max;
+    }
+
+    public int getMin() {
+      return min;
+    }
+    public int getMax() {
+      return max;
+    }
+
+    @Override
+    public Void apply(Void aVoid) {
+      int sleepMs = ThreadLocalRandom.current().nextInt(min, max + 1);
+      try {
+        Thread.sleep(sleepMs);
+      } catch (InterruptedException e) {
+      }
+      return null;
+    }
+  }
+
+  public static class FixedLatency implements Function<Void, Void> {
+    private int latency;
+    public FixedLatency(int latency) {
+      this.latency = latency;
+    }
+    public int getLatency() {
+      return latency;
+    }
+
+    @Override
+    public Void apply(Void aVoid) {
+      if(latency > 0) {
+        try {
+          Thread.sleep(latency);
+        } catch (InterruptedException e) {
+        }
+      }
+      return null;
+    }
+  }
+  Function<Void, Void> sleepFunction = null;
+
+  public NoopWriter withLatency(String sleepConfig) {
+    sleepFunction = getSleepFunction(sleepConfig);
+    return this;
+  }
+
+
+  private Function<Void, Void> getSleepFunction(String sleepConfig) {
+    String usageMessage = "Unexpected: " + sleepConfig + " Expected value: integer for a fixed sleep duration in milliseconds (e.g. 10) " +
+            "or a range of latencies separated by a comma (e.g. \"10, 20\") to sleep a random amount in that range.";
+    try {
+      if (sleepConfig.contains(",")) {
+        // random latency within a range.
+        Iterable<String> it = Splitter.on(',').split(sleepConfig);
+        Integer min = ConversionUtils.convert(Iterables.getFirst(it, "").trim(), Integer.class);
+        Integer max= ConversionUtils.convert(Iterables.getLast(it, "").trim(), Integer.class);
+        if (min != null && max != null) {
+          return new RandomLatency(min, max);
+        }
+      } else {
+        //fixed latency
+        Integer latency = ConversionUtils.convert(sleepConfig.trim(), Integer.class);
+        if(latency != null) {
+          return new FixedLatency(latency);
+        }
+      }
+    }
+    catch(Throwable t) {
+      throw new IllegalArgumentException(usageMessage, t);
+    }
+    throw new IllegalArgumentException(usageMessage);
+  }
+
+  @Override
+  public void configure(String sensorName, WriterConfiguration configuration) {
+    Map<String, Object> config = configuration.getSensorConfig(sensorName);
+    if(config != null) {
+      Object noopLatency = config.get("noopLatency");
+      if(noopLatency != null) {
+        sleepFunction = getSleepFunction(noopLatency.toString());
+      }
+    }
+  }
+
+  @Override
+  public void init(Map stormConf, WriterConfiguration config) throws Exception {
+  }
+
+  @Override
+  public void write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<Object> messages) throws Exception {
+    if(sleepFunction != null) {
+      sleepFunction.apply(null);
+    }
+  }
+
+  @Override
+  public void close() throws Exception {
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java
new file mode 100644
index 0000000..3df3e9d
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer;
+
+import backtype.storm.tuple.Tuple;
+import com.google.common.collect.Iterables;
+import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFacade;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.interfaces.MessageWriter;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+public class WriterToBulkWriter<MESSAGE_T> implements BulkMessageWriter<MESSAGE_T>, Serializable {
+  MessageWriter<MESSAGE_T> messageWriter;
+
+  public static transient Function<WriterConfiguration, WriterConfiguration> TRANSFORMATION = config -> new SingleBatchConfigurationFacade(config);
+
+  public WriterToBulkWriter(MessageWriter<MESSAGE_T> messageWriter) {
+    this.messageWriter = messageWriter;
+  }
+  @Override
+  public void init(Map stormConf, WriterConfiguration config) throws Exception {
+    messageWriter.init();
+  }
+
+  @Override
+  public void write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<MESSAGE_T> messages) throws Exception {
+    if(messages.size() > 1) {
+      throw new IllegalStateException("WriterToBulkWriter expects a batch of exactly 1");
+    }
+    messageWriter.write(sensorType, configurations, Iterables.getFirst(tuples, null), Iterables.getFirst(messages, null));
+  }
+
+  @Override
+  public void close() throws Exception {
+    messageWriter.close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
new file mode 100644
index 0000000..91e8446
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer.bolt;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt;
+import org.apache.metron.common.configuration.writer.EnrichmentWriterConfiguration;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.MessageWriter;
+import org.apache.metron.common.utils.MessageUtils;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.writer.BulkWriterComponent;
+import org.apache.metron.writer.WriterToBulkWriter;
+import org.apache.metron.writer.message.MessageGetter;
+import org.apache.metron.writer.message.MessageGetters;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.function.Function;
+
+public class BulkMessageWriterBolt extends ConfiguredEnrichmentBolt {
+
+  private static final Logger LOG = LoggerFactory
+          .getLogger(BulkMessageWriterBolt.class);
+  private BulkMessageWriter<JSONObject> bulkMessageWriter;
+  private BulkWriterComponent<JSONObject> writerComponent;
+  private String messageGetterStr = MessageGetters.NAMED.name();
+  private transient MessageGetter messageGetter = null;
+  private transient Function<WriterConfiguration, WriterConfiguration> configurationTransformation;
+  public BulkMessageWriterBolt(String zookeeperUrl) {
+    super(zookeeperUrl);
+  }
+
+  public BulkMessageWriterBolt withBulkMessageWriter(BulkMessageWriter<JSONObject > bulkMessageWriter) {
+    this.bulkMessageWriter = bulkMessageWriter;
+    return this;
+  }
+
+  public BulkMessageWriterBolt withMessageWriter(MessageWriter<JSONObject> messageWriter) {
+    this.bulkMessageWriter = new WriterToBulkWriter<>(messageWriter);
+    return this;
+  }
+
+  public BulkMessageWriterBolt withMessageGetter(String messageGetter) {
+    this.messageGetterStr = messageGetter;
+    return this;
+  }
+
+  @Override
+  public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+    this.writerComponent = new BulkWriterComponent<>(collector);
+    super.prepare(stormConf, context, collector);
+    messageGetter = MessageGetters.valueOf(messageGetterStr);
+    if(bulkMessageWriter instanceof WriterToBulkWriter) {
+      configurationTransformation = WriterToBulkWriter.TRANSFORMATION;
+    }
+    else {
+      configurationTransformation = x -> x;
+    }
+    try {
+      bulkMessageWriter.init(stormConf
+                            , configurationTransformation.apply(new EnrichmentWriterConfiguration(getConfigurations()))
+                            );
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  @Override
+  public void execute(Tuple tuple) {
+    JSONObject message = messageGetter.getMessage(tuple);
+    String sensorType = MessageUtils.getSensorType(message);
+    try
+    {
+      writerComponent.write(sensorType
+                           , tuple
+                           , message
+                           , bulkMessageWriter
+                           , configurationTransformation.apply(new EnrichmentWriterConfiguration(getConfigurations()))
+                           );
+    }
+    catch(Exception e) {
+      throw new RuntimeException("This should have been caught in the writerComponent.  If you see this, file a JIRA", e);
+    }
+  }
+
+  @Override
+  public void declareOutputFields(OutputFieldsDeclarer declarer) {
+    declarer.declareStream(Constants.ERROR_STREAM, new Fields("message"));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
new file mode 100644
index 0000000..01f1245
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer.hdfs;
+
+import backtype.storm.tuple.Tuple;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.storm.hdfs.bolt.format.FileNameFormat;
+import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
+import org.apache.storm.hdfs.bolt.rotation.NoRotationPolicy;
+import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
+import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
+import org.apache.storm.hdfs.common.rotation.RotationAction;
+import org.json.simple.JSONObject;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class HdfsWriter implements BulkMessageWriter<JSONObject>, Serializable {
+  List<RotationAction> rotationActions = new ArrayList<>();
+  FileRotationPolicy rotationPolicy = new NoRotationPolicy();
+  SyncPolicy syncPolicy = new CountSyncPolicy(1); //sync every time, duh.
+  FileNameFormat fileNameFormat;
+  Map<String, SourceHandler> sourceHandlerMap = new HashMap<>();
+  transient Map stormConfig;
+  public HdfsWriter withFileNameFormat(FileNameFormat fileNameFormat){
+    this.fileNameFormat = fileNameFormat;
+    return this;
+  }
+
+  public HdfsWriter withSyncPolicy(SyncPolicy syncPolicy){
+    this.syncPolicy = syncPolicy;
+    return this;
+  }
+  public HdfsWriter withRotationPolicy(FileRotationPolicy rotationPolicy){
+    this.rotationPolicy = rotationPolicy;
+    return this;
+  }
+
+  public HdfsWriter addRotationAction(RotationAction action){
+    this.rotationActions.add(action);
+    return this;
+  }
+
+  @Override
+  public void init(Map stormConfig, WriterConfiguration configurations) {
+    this.stormConfig = stormConfig;
+  }
+
+
+  @Override
+  public void write(String sourceType
+                   , WriterConfiguration configurations
+                   , Iterable<Tuple> tuples
+                   , List<JSONObject> messages
+                   ) throws Exception
+  {
+    SourceHandler handler = getSourceHandler(sourceType);
+    handler.handle(messages);
+  }
+
+  @Override
+  public void close() {
+    for(SourceHandler handler : sourceHandlerMap.values()) {
+      handler.close();
+    }
+  }
+  private synchronized SourceHandler getSourceHandler(String sourceType) throws IOException {
+    SourceHandler ret = sourceHandlerMap.get(sourceType);
+    if(ret == null) {
+      ret = new SourceHandler(rotationActions, rotationPolicy, syncPolicy, new SourceFileNameFormat(sourceType, fileNameFormat), stormConfig);
+      sourceHandlerMap.put(sourceType, ret);
+    }
+    return ret;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
new file mode 100644
index 0000000..1c345b4
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.hdfs;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+import org.apache.storm.hdfs.common.rotation.RotationAction;
+
+import java.io.IOException;
+
+public class SourceAwareMoveAction implements RotationAction{
+  private static final Logger LOG = Logger.getLogger(SourceHandler.class);
+  private String destination;
+
+  public SourceAwareMoveAction toDestination(String destDir){
+    destination = destDir;
+    return this;
+  }
+
+  private static String getSource(Path filePath) {
+    return filePath.getParent().getName();
+  }
+
+  @Override
+  public void execute(FileSystem fileSystem, Path filePath) throws IOException {
+    Path destPath = new Path(new Path(destination, getSource(filePath)), filePath.getName());
+    LOG.info("Moving file " + filePath + " to " + destPath);
+    boolean success = fileSystem.rename(filePath, destPath);
+    return;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java
new file mode 100644
index 0000000..ae0242d
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.hdfs;
+
+import backtype.storm.task.TopologyContext;
+import org.apache.storm.hdfs.bolt.format.FileNameFormat;
+
+import java.util.Map;
+
+public class SourceFileNameFormat implements FileNameFormat {
+  FileNameFormat delegate;
+  String sourceType;
+  public SourceFileNameFormat(String sourceType, FileNameFormat delegate) {
+    this.delegate = delegate;
+    this.sourceType = sourceType;
+  }
+
+  @Override
+  public void prepare(Map map, TopologyContext topologyContext) {
+    this.delegate.prepare(map, topologyContext);
+  }
+
+  @Override
+  public String getName(long l, long l1) {
+    return delegate.getName(l, l1);
+  }
+
+  @Override
+  public String getPath() {
+    return delegate.getPath() + "/" + sourceType;
+  }
+}


[4/5] incubator-metron git commit: METRON-154: Decouple enrichment and indexing closes apache/incubator-metron#192

Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml
index 226b686..0f391c4 100644
--- a/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml
+++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml
@@ -21,7 +21,7 @@ config:
 components:
 # Enrichment
     -   id: "geoEnrichmentAdapter"
-        className: "org.apache.metron.integration.mock.MockGeoAdapter"
+        className: "org.apache.metron.enrichment.integration.mock.MockGeoAdapter"
     -   id: "geoEnrichment"
         className: "org.apache.metron.enrichment.configuration.Enrichment"
         constructorArgs:
@@ -111,27 +111,17 @@ components:
                 args:
                     - ref: "simpleHBaseThreatIntelEnrichment"
 
-    -   id: "fileNameFormat"
-        className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat"
-        configMethods:
-            -   name: "withPrefix"
-                args:
-                    - "enrichment-"
-            -   name: "withExtension"
-                args:
-                  - ".json"
-            -   name: "withPath"
-                args:
-                    - "${index.hdfs.output}"
+
 #indexing
-    -   id: "hdfsWriter"
-        className: "org.apache.metron.writer.hdfs.HdfsWriter"
+    -   id: "kafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
         configMethods:
-            -   name: "withFileNameFormat"
+            -   name: "withTopic"
                 args:
-                    - ref: "fileNameFormat"
-    -   id: "indexWriter"
-        className: "${writer.class.name}"
+                    - "${enrichment.output.topic}"
+            -   name: "withZkQuorum"
+                args:
+                    - "${kafka.zk}"
 
 #kafka/zookeeper
     -   id: "zkHosts"
@@ -249,22 +239,14 @@ bolts:
             -   name: "withMaxTimeRetain"
                 args: [10]
 # Indexing Bolts
-    -   id: "indexingBolt"
-        className: "org.apache.metron.enrichment.bolt.BulkMessageWriterBolt"
-        constructorArgs:
-            - "${kafka.zk}"
-        configMethods:
-            -   name: "withBulkMessageWriter"
-                args:
-                    - ref: "indexWriter"
-    -   id: "hdfsIndexingBolt"
-        className: "org.apache.metron.enrichment.bolt.BulkMessageWriterBolt"
+    -   id: "outputBolt"
+        className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
         constructorArgs:
             - "${kafka.zk}"
         configMethods:
-            -   name: "withBulkMessageWriter"
+            -   name: "withMessageWriter"
                 args:
-                    - ref: "hdfsWriter"
+                    - ref: "kafkaWriter"
 
 
 streams:
@@ -361,24 +343,12 @@ streams:
             streamId: "message"
             type: FIELDS
             args: ["key"]
-#indexing
-    -   name: "threatIntelJoin -> indexing"
+
+#output
+    -   name: "threatIntelJoin -> output"
         from: "threatIntelJoinBolt"
-        to: "indexingBolt"
+        to: "outputBolt"
         grouping:
             streamId: "message"
             type: FIELDS
             args: ["key"]
-    -   name: "threatIntelJoin -> hdfs"
-        from: "threatIntelJoinBolt"
-        to: "hdfsIndexingBolt"
-        grouping:
-            streamId: "message"
-            type: SHUFFLE
-
-    -   name: "indexingBolt -> errorIndexingBolt"
-        from: "indexingBolt"
-        to: "indexingBolt"
-        grouping:
-            streamId: "error"
-            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
deleted file mode 100644
index 772dfc6..0000000
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.enrichment.bolt;
-
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Fields;
-import backtype.storm.tuple.Tuple;
-import org.apache.metron.common.Constants;
-import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt;
-import org.apache.metron.common.configuration.writer.EnrichmentWriterConfiguration;
-import org.apache.metron.common.utils.ErrorUtils;
-import org.apache.metron.common.utils.MessageUtils;
-import org.apache.metron.common.interfaces.BulkMessageWriter;
-import org.apache.metron.common.writer.BulkWriterComponent;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-
-public class BulkMessageWriterBolt extends ConfiguredEnrichmentBolt {
-
-  private static final Logger LOG = LoggerFactory
-          .getLogger(BulkMessageWriterBolt.class);
-  private BulkMessageWriter<JSONObject> bulkMessageWriter;
-  private BulkWriterComponent<JSONObject> writerComponent;
-  public BulkMessageWriterBolt(String zookeeperUrl) {
-    super(zookeeperUrl);
-  }
-
-  public BulkMessageWriterBolt withBulkMessageWriter(BulkMessageWriter<JSONObject > bulkMessageWriter) {
-    this.bulkMessageWriter = bulkMessageWriter;
-    return this;
-  }
-
-  @Override
-  public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
-    this.writerComponent = new BulkWriterComponent<>(collector);
-    super.prepare(stormConf, context, collector);
-    try {
-      bulkMessageWriter.init(stormConf, new EnrichmentWriterConfiguration(getConfigurations()));
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-
-  @SuppressWarnings("unchecked")
-  @Override
-  public void execute(Tuple tuple) {
-    JSONObject message =(JSONObject)tuple.getValueByField("message");
-    String sensorType = MessageUtils.getSensorType(message);
-    try
-    {
-      writerComponent.write(sensorType, tuple, message, bulkMessageWriter, new EnrichmentWriterConfiguration(getConfigurations()));
-    }
-    catch(Exception e) {
-      throw new RuntimeException("This should have been caught in the writerComponent.  If you see this, file a JIRA", e);
-    }
-  }
-
-  @Override
-  public void declareOutputFields(OutputFieldsDeclarer declarer) {
-    declarer.declareStream(Constants.ERROR_STREAM, new Fields("message"));
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java
index 3c38824..3b7e437 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java
@@ -17,13 +17,12 @@
  */
 package org.apache.metron.enrichment.lookup.accesstracker;
 
-import com.google.common.hash.BloomFilter;
-import com.google.common.hash.Funnel;
-import com.google.common.hash.PrimitiveSink;
+import org.apache.metron.common.utils.BloomFilter;
 import org.apache.metron.enrichment.lookup.LookupKey;
 
 import java.io.*;
 import java.util.Map;
+import java.util.function.Function;
 
 public class BloomAccessTracker implements AccessTracker {
     private static final long serialVersionUID = 1L;
@@ -31,25 +30,14 @@ public class BloomAccessTracker implements AccessTracker {
     public static final String FALSE_POSITIVE_RATE_KEY = "false_positive_rate";
     public static final String NAME_KEY = "name";
 
-    private static class LookupKeyFunnel implements Funnel<LookupKey> {
-        @Override
-        public void funnel(LookupKey lookupKey, PrimitiveSink primitiveSink) {
-            primitiveSink.putBytes(lookupKey.toBytes());
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            return this.getClass().equals(obj.getClass());
-        }
 
+    public static class LookupKeySerializer implements Function<LookupKey, byte[]>, Serializable {
         @Override
-        public int hashCode() {
-            return super.hashCode() * 31;
+        public byte[] apply(LookupKey lookupKey) {
+            return lookupKey.toBytes();
         }
     }
 
-    private static Funnel<LookupKey> LOOKUPKEY_FUNNEL = new LookupKeyFunnel();
-
     BloomFilter<LookupKey> filter;
     String name;
     int expectedInsertions;
@@ -60,7 +48,7 @@ public class BloomAccessTracker implements AccessTracker {
         this.name = name;
         this.expectedInsertions = expectedInsertions;
         this.falsePositiveRate = falsePositiveRate;
-        filter = BloomFilter.create(LOOKUPKEY_FUNNEL, expectedInsertions, falsePositiveRate);
+        filter = new BloomFilter<LookupKey>(new LookupKeySerializer(), expectedInsertions, falsePositiveRate);
     }
     public BloomAccessTracker() {}
     public BloomAccessTracker(Map<String, Object> config) {
@@ -73,7 +61,7 @@ public class BloomAccessTracker implements AccessTracker {
     @Override
     public void logAccess(LookupKey key) {
         numInsertions++;
-        filter.put(key);
+        filter.add(key);
     }
 
     @Override
@@ -81,7 +69,7 @@ public class BloomAccessTracker implements AccessTracker {
         expectedInsertions = toInt(config.get(EXPECTED_INSERTIONS_KEY));
         falsePositiveRate = toDouble(config.get(FALSE_POSITIVE_RATE_KEY));
         name = config.get(NAME_KEY).toString();
-        filter = BloomFilter.create(LOOKUPKEY_FUNNEL, expectedInsertions, falsePositiveRate);
+        filter = new BloomFilter<LookupKey>(new LookupKeySerializer(), expectedInsertions, falsePositiveRate);
     }
 
     @Override
@@ -91,7 +79,7 @@ public class BloomAccessTracker implements AccessTracker {
 
     @Override
     public void reset() {
-        filter = BloomFilter.create(LOOKUPKEY_FUNNEL, expectedInsertions, falsePositiveRate);
+        filter = new BloomFilter<LookupKey>(new LookupKeySerializer(), expectedInsertions, falsePositiveRate);
     }
 
     private static double toDouble(Object o) {
@@ -129,7 +117,7 @@ public class BloomAccessTracker implements AccessTracker {
             throw new IllegalStateException("Unable to union access tracker, because this tracker is not initialized.");
         }
         if(tracker instanceof BloomAccessTracker ) {
-            filter.putAll(((BloomAccessTracker)tracker).getFilter());
+            filter.merge(((BloomAccessTracker)tracker).getFilter());
             return this;
         }
         else {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java
new file mode 100644
index 0000000..cfdae33
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java
@@ -0,0 +1,275 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.enrichment.writer;
+
+import backtype.storm.tuple.Tuple;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.common.utils.ReflectionUtils;
+import org.apache.metron.enrichment.converter.EnrichmentConverter;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.hbase.HTableProvider;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.writer.AbstractWriter;
+import org.json.simple.JSONObject;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class SimpleHbaseEnrichmentWriter extends AbstractWriter implements BulkMessageWriter<JSONObject>, Serializable {
+  public enum Configurations {
+    HBASE_TABLE("shew.table")
+    ,HBASE_CF("shew.cf")
+    ,KEY_COLUMNS("shew.keyColumns")
+    ,KEY_DELIM("shew.keyDelim")
+    ,ENRICHMENT_TYPE("shew.enrichmentType")
+    ,VALUE_COLUMNS("shew.valueColumns")
+    ,HBASE_PROVIDER("shew.hbaseProvider")
+    ;
+    String key;
+    Configurations(String key) {
+      this.key = key;
+    }
+    public String getKey() {
+      return key;
+    }
+    public Object get(Map<String, Object> config) {
+      return config.get(key);
+    }
+    public <T> T getAndConvert(Map<String, Object> config, Class<T> clazz) {
+      Object o = get(config);
+      if(o != null) {
+        return ConversionUtils.convert(o, clazz);
+      }
+      return null;
+    }
+  }
+  public static class KeyTransformer {
+    List<String> keys = new ArrayList<>();
+    Set<String> keySet;
+    private String delim = ":";
+    public KeyTransformer(String key) {
+      this(key, null);
+    }
+    public KeyTransformer(String key, String delim) {
+      keys.add(key);
+      keySet = new HashSet<>(this.keys);
+      this.delim = delim == null?this.delim:delim;
+    }
+    public KeyTransformer(Iterable<String> keys) {
+      this(keys, null);
+    }
+    public KeyTransformer(Iterable<String> keys, String delim) {
+      Iterables.addAll(this.keys, keys);
+      keySet = new HashSet<>(this.keys);
+      this.delim = delim == null?this.delim:delim;
+    }
+
+    public String transform(final JSONObject message) {
+      return
+      keys.stream().map( x -> {
+        Object o = message.get(x);
+        return o == null?"":o.toString();
+      }).collect(Collectors.joining(delim));
+    }
+  }
+  private transient EnrichmentConverter converter;
+  private String tableName;
+  private String cf;
+  private HTableInterface table;
+  private TableProvider provider;
+  private Map.Entry<Object, KeyTransformer> keyTransformer;
+
+  public SimpleHbaseEnrichmentWriter() {
+  }
+
+  @Override
+  public void configure(String sensorName, WriterConfiguration configuration) {
+    String hbaseProviderImpl = Configurations.HBASE_PROVIDER.getAndConvert(configuration.getSensorConfig(sensorName),String.class);
+    if(hbaseProviderImpl != null) {
+      provider = ReflectionUtils.createInstance(hbaseProviderImpl);
+    }
+    if(converter == null) {
+      converter = new EnrichmentConverter();
+    }
+  }
+
+  @Override
+  public void init(Map stormConf, WriterConfiguration configuration) throws Exception {
+    if(converter == null) {
+      converter = new EnrichmentConverter();
+    }
+  }
+
+  protected synchronized TableProvider getProvider() {
+    if(provider == null) {
+
+      provider = new HTableProvider();
+    }
+    return provider;
+  }
+
+  public HTableInterface getTable(String tableName, String cf) throws IOException {
+    synchronized(this) {
+      boolean isInitial = this.tableName == null || this.cf == null;
+      boolean isValid = tableName != null && cf != null;
+
+      if( isInitial || (isValid && (!this.tableName.equals(tableName) || !this.cf.equals(cf)) )
+        )
+      {
+        Configuration conf = HBaseConfiguration.create();
+        //new table connection
+        if(table != null) {
+          table.close();
+        }
+        table = getProvider().getTable(conf, tableName);
+        this.tableName = tableName;
+        this.cf = cf;
+      }
+      return table;
+    }
+  }
+
+  public HTableInterface getTable(Map<String, Object> config) throws IOException {
+    return getTable(Configurations.HBASE_TABLE.getAndConvert(config, String.class)
+                   ,Configurations.HBASE_CF.getAndConvert(config, String.class)
+                   );
+
+  }
+
+
+  private List<String> getColumns(Object keyColumnsObj, boolean allowNull) {
+    Object o = keyColumnsObj;
+    if(allowNull && keyColumnsObj == null) {
+      return Collections.emptyList();
+    }
+    if(o instanceof String) {
+      return ImmutableList.of(o.toString());
+    }
+    else if (o instanceof List) {
+      List<String> keyCols = new ArrayList<>();
+      for(Object key : (List)o) {
+        keyCols.add(key.toString());
+      }
+      return keyCols;
+    }
+    else {
+      throw new RuntimeException("Unable to get columns: " + o);
+    }
+  }
+
+  private KeyTransformer getTransformer(Map<String, Object> config) {
+    Object o = Configurations.KEY_COLUMNS.get(config);
+    KeyTransformer transformer = null;
+    if(keyTransformer != null && keyTransformer.getKey() == o) {
+      return keyTransformer.getValue();
+    }
+    else {
+      List<String> keys = getColumns(o, false);
+      Object delimObj = Configurations.KEY_DELIM.get(config);
+      String delim = (delimObj == null || !(delimObj instanceof String))?null:delimObj.toString();
+      transformer = new KeyTransformer(keys, delim);
+      keyTransformer = new AbstractMap.SimpleEntry<>(o, transformer);
+      return transformer;
+    }
+  }
+
+
+  private EnrichmentValue getValue( JSONObject message
+                                  , Set<String> keyColumns
+                                  , Set<String> valueColumns
+                                  )
+  {
+    Map<String, Object> metadata = new HashMap<>();
+    if(valueColumns == null || valueColumns.isEmpty()) {
+      for (Object kv : message.entrySet()) {
+        Map.Entry<Object, Object> entry = (Map.Entry<Object, Object>) kv;
+        if (!keyColumns.contains(entry.getKey())) {
+          metadata.put(entry.getKey().toString(), entry.getValue());
+        }
+      }
+      return new EnrichmentValue(metadata);
+    }
+    else {
+      for (Object kv : message.entrySet()) {
+        Map.Entry<Object, Object> entry = (Map.Entry<Object, Object>) kv;
+        if (valueColumns.contains(entry.getKey())) {
+          metadata.put(entry.getKey().toString(), entry.getValue());
+        }
+      }
+      return new EnrichmentValue(metadata);
+    }
+  }
+
+  private EnrichmentKey getKey(JSONObject message, KeyTransformer transformer, String enrichmentType) {
+    if(enrichmentType != null) {
+      return new EnrichmentKey(enrichmentType, transformer.transform(message));
+    }
+    else {
+      return null;
+    }
+  }
+
+  @Override
+  public void write( String sensorType
+                    , WriterConfiguration configurations
+                    , Iterable<Tuple> tuples
+                    , List<JSONObject> messages
+                    ) throws Exception
+  {
+    Map<String, Object> sensorConfig = configurations.getSensorConfig(sensorType);
+    HTableInterface table = getTable(sensorConfig);
+    KeyTransformer transformer = getTransformer(sensorConfig);
+    Object enrichmentTypeObj = Configurations.ENRICHMENT_TYPE.get(sensorConfig);
+    String enrichmentType = enrichmentTypeObj == null?null:enrichmentTypeObj.toString();
+    Set<String> valueColumns = new HashSet<>(getColumns(Configurations.VALUE_COLUMNS.get(sensorConfig), true));
+    List<Put> puts = new ArrayList<>();
+    for(JSONObject message : messages) {
+      EnrichmentKey key = getKey(message, transformer, enrichmentType);
+      EnrichmentValue value = getValue(message, transformer.keySet, valueColumns);
+      if(key == null || value == null) {
+        continue;
+      }
+      Put put = converter.toPut(this.cf, key, value);
+      if(put != null) {
+        puts.add(put);
+      }
+    }
+    table.put(puts);
+  }
+
+  @Override
+  public void close() throws Exception {
+    synchronized(this) {
+      if(table != null) {
+        table.close();
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
deleted file mode 100644
index f7e7aeb..0000000
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
+++ /dev/null
@@ -1,282 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.writer.hbase;
-
-import backtype.storm.tuple.Tuple;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.common.configuration.ParserConfigurations;
-import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.interfaces.BulkMessageWriter;
-import org.apache.metron.common.utils.ConversionUtils;
-import org.apache.metron.common.utils.ReflectionUtils;
-import org.apache.metron.common.writer.AbstractWriter;
-import org.apache.metron.enrichment.converter.EnrichmentConverter;
-import org.apache.metron.enrichment.converter.EnrichmentKey;
-import org.apache.metron.enrichment.converter.EnrichmentValue;
-import org.apache.metron.enrichment.converter.HbaseConverter;
-import org.apache.metron.hbase.HTableProvider;
-import org.apache.metron.hbase.TableProvider;
-import org.json.simple.JSONObject;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.*;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-public class SimpleHbaseEnrichmentWriter extends AbstractWriter implements BulkMessageWriter<JSONObject>, Serializable {
-  public enum Configurations {
-    HBASE_TABLE("shew.table")
-    ,HBASE_CF("shew.cf")
-    ,KEY_COLUMNS("shew.keyColumns")
-    ,KEY_DELIM("shew.keyDelim")
-    ,ENRICHMENT_TYPE("shew.enrichmentType")
-    ,VALUE_COLUMNS("shew.valueColumns")
-    ,HBASE_PROVIDER("shew.hbaseProvider")
-    ;
-    String key;
-    Configurations(String key) {
-      this.key = key;
-    }
-    public String getKey() {
-      return key;
-    }
-    public Object get(Map<String, Object> config) {
-      return config.get(key);
-    }
-    public <T> T getAndConvert(Map<String, Object> config, Class<T> clazz) {
-      Object o = get(config);
-      if(o != null) {
-        return ConversionUtils.convert(o, clazz);
-      }
-      return null;
-    }
-  }
-  public static class KeyTransformer {
-    List<String> keys = new ArrayList<>();
-    Set<String> keySet;
-    private String delim = ":";
-    public KeyTransformer(String key) {
-      this(key, null);
-    }
-    public KeyTransformer(String key, String delim) {
-      keys.add(key);
-      keySet = new HashSet<>(this.keys);
-      this.delim = delim == null?this.delim:delim;
-    }
-    public KeyTransformer(Iterable<String> keys) {
-      this(keys, null);
-    }
-    public KeyTransformer(Iterable<String> keys, String delim) {
-      Iterables.addAll(this.keys, keys);
-      keySet = new HashSet<>(this.keys);
-      this.delim = delim == null?this.delim:delim;
-    }
-
-    public String transform(final JSONObject message) {
-      return
-      keys.stream().map( x -> {
-        Object o = message.get(x);
-        return o == null?"":o.toString();
-      }).collect(Collectors.joining(delim));
-    }
-  }
-  private transient EnrichmentConverter converter;
-  private String tableName;
-  private String cf;
-  private HTableInterface table;
-  private TableProvider provider;
-  private Map.Entry<Object, KeyTransformer> keyTransformer;
-
-  public SimpleHbaseEnrichmentWriter() {
-  }
-
-  @Override
-  public void configure(String sensorName, WriterConfiguration configuration) {
-    String hbaseProviderImpl = Configurations.HBASE_PROVIDER.getAndConvert(configuration.getSensorConfig(sensorName),String.class);
-    if(hbaseProviderImpl != null) {
-      provider = ReflectionUtils.createInstance(hbaseProviderImpl);
-    }
-    if(converter == null) {
-      converter = new EnrichmentConverter();
-    }
-  }
-
-  @Override
-  public void init(Map stormConf, WriterConfiguration configuration) throws Exception {
-    if(converter == null) {
-      converter = new EnrichmentConverter();
-    }
-  }
-
-  protected synchronized TableProvider getProvider() {
-    if(provider == null) {
-
-      provider = new HTableProvider();
-    }
-    return provider;
-  }
-
-  public HTableInterface getTable(String tableName, String cf) throws IOException {
-    synchronized(this) {
-      boolean isInitial = this.tableName == null || this.cf == null;
-      boolean isValid = tableName != null && cf != null;
-
-      if( isInitial || (isValid && (!this.tableName.equals(tableName) || !this.cf.equals(cf)) )
-        )
-      {
-        Configuration conf = HBaseConfiguration.create();
-        //new table connection
-        if(table != null) {
-          table.close();
-        }
-        table = getProvider().getTable(conf, tableName);
-        this.tableName = tableName;
-        this.cf = cf;
-      }
-      return table;
-    }
-  }
-
-  public HTableInterface getTable(Map<String, Object> config) throws IOException {
-    return getTable(Configurations.HBASE_TABLE.getAndConvert(config, String.class)
-                   ,Configurations.HBASE_CF.getAndConvert(config, String.class)
-                   );
-
-  }
-
-
-  private List<String> getColumns(Object keyColumnsObj, boolean allowNull) {
-    Object o = keyColumnsObj;
-    if(allowNull && keyColumnsObj == null) {
-      return Collections.emptyList();
-    }
-    if(o instanceof String) {
-      return ImmutableList.of(o.toString());
-    }
-    else if (o instanceof List) {
-      List<String> keyCols = new ArrayList<>();
-      for(Object key : (List)o) {
-        keyCols.add(key.toString());
-      }
-      return keyCols;
-    }
-    else {
-      throw new RuntimeException("Unable to get columns: " + o);
-    }
-  }
-
-  private KeyTransformer getTransformer(Map<String, Object> config) {
-    Object o = Configurations.KEY_COLUMNS.get(config);
-    KeyTransformer transformer = null;
-    if(keyTransformer != null && keyTransformer.getKey() == o) {
-      return keyTransformer.getValue();
-    }
-    else {
-      List<String> keys = getColumns(o, false);
-      Object delimObj = Configurations.KEY_DELIM.get(config);
-      String delim = (delimObj == null || !(delimObj instanceof String))?null:delimObj.toString();
-      transformer = new KeyTransformer(keys, delim);
-      keyTransformer = new AbstractMap.SimpleEntry<>(o, transformer);
-      return transformer;
-    }
-  }
-
-
-  private EnrichmentValue getValue( JSONObject message
-                                  , Set<String> keyColumns
-                                  , Set<String> valueColumns
-                                  )
-  {
-    Map<String, Object> metadata = new HashMap<>();
-    if(valueColumns == null || valueColumns.isEmpty()) {
-      for (Object kv : message.entrySet()) {
-        Map.Entry<Object, Object> entry = (Map.Entry<Object, Object>) kv;
-        if (!keyColumns.contains(entry.getKey())) {
-          metadata.put(entry.getKey().toString(), entry.getValue());
-        }
-      }
-      return new EnrichmentValue(metadata);
-    }
-    else {
-      for (Object kv : message.entrySet()) {
-        Map.Entry<Object, Object> entry = (Map.Entry<Object, Object>) kv;
-        if (valueColumns.contains(entry.getKey())) {
-          metadata.put(entry.getKey().toString(), entry.getValue());
-        }
-      }
-      return new EnrichmentValue(metadata);
-    }
-  }
-
-  private EnrichmentKey getKey(JSONObject message, KeyTransformer transformer, String enrichmentType) {
-    if(enrichmentType != null) {
-      return new EnrichmentKey(enrichmentType, transformer.transform(message));
-    }
-    else {
-      return null;
-    }
-  }
-
-  @Override
-  public void write( String sensorType
-                    , WriterConfiguration configurations
-                    , Iterable<Tuple> tuples
-                    , List<JSONObject> messages
-                    ) throws Exception
-  {
-    Map<String, Object> sensorConfig = configurations.getSensorConfig(sensorType);
-    HTableInterface table = getTable(sensorConfig);
-    KeyTransformer transformer = getTransformer(sensorConfig);
-    Object enrichmentTypeObj = Configurations.ENRICHMENT_TYPE.get(sensorConfig);
-    String enrichmentType = enrichmentTypeObj == null?null:enrichmentTypeObj.toString();
-    Set<String> valueColumns = new HashSet<>(getColumns(Configurations.VALUE_COLUMNS.get(sensorConfig), true));
-    List<Put> puts = new ArrayList<>();
-    for(JSONObject message : messages) {
-      EnrichmentKey key = getKey(message, transformer, enrichmentType);
-      EnrichmentValue value = getValue(message, transformer.keySet, valueColumns);
-      if(key == null || value == null) {
-        continue;
-      }
-      Put put = converter.toPut(this.cf, key, value);
-      if(put != null) {
-        puts.add(put);
-      }
-    }
-    table.put(puts);
-  }
-
-  @Override
-  public void close() throws Exception {
-    synchronized(this) {
-      if(table != null) {
-        table.close();
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
deleted file mode 100644
index 01f1245..0000000
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.writer.hdfs;
-
-import backtype.storm.tuple.Tuple;
-import org.apache.metron.common.configuration.EnrichmentConfigurations;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.interfaces.BulkMessageWriter;
-import org.apache.storm.hdfs.bolt.format.FileNameFormat;
-import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
-import org.apache.storm.hdfs.bolt.rotation.NoRotationPolicy;
-import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
-import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
-import org.apache.storm.hdfs.common.rotation.RotationAction;
-import org.json.simple.JSONObject;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class HdfsWriter implements BulkMessageWriter<JSONObject>, Serializable {
-  List<RotationAction> rotationActions = new ArrayList<>();
-  FileRotationPolicy rotationPolicy = new NoRotationPolicy();
-  SyncPolicy syncPolicy = new CountSyncPolicy(1); //sync every time, duh.
-  FileNameFormat fileNameFormat;
-  Map<String, SourceHandler> sourceHandlerMap = new HashMap<>();
-  transient Map stormConfig;
-  public HdfsWriter withFileNameFormat(FileNameFormat fileNameFormat){
-    this.fileNameFormat = fileNameFormat;
-    return this;
-  }
-
-  public HdfsWriter withSyncPolicy(SyncPolicy syncPolicy){
-    this.syncPolicy = syncPolicy;
-    return this;
-  }
-  public HdfsWriter withRotationPolicy(FileRotationPolicy rotationPolicy){
-    this.rotationPolicy = rotationPolicy;
-    return this;
-  }
-
-  public HdfsWriter addRotationAction(RotationAction action){
-    this.rotationActions.add(action);
-    return this;
-  }
-
-  @Override
-  public void init(Map stormConfig, WriterConfiguration configurations) {
-    this.stormConfig = stormConfig;
-  }
-
-
-  @Override
-  public void write(String sourceType
-                   , WriterConfiguration configurations
-                   , Iterable<Tuple> tuples
-                   , List<JSONObject> messages
-                   ) throws Exception
-  {
-    SourceHandler handler = getSourceHandler(sourceType);
-    handler.handle(messages);
-  }
-
-  @Override
-  public void close() {
-    for(SourceHandler handler : sourceHandlerMap.values()) {
-      handler.close();
-    }
-  }
-  private synchronized SourceHandler getSourceHandler(String sourceType) throws IOException {
-    SourceHandler ret = sourceHandlerMap.get(sourceType);
-    if(ret == null) {
-      ret = new SourceHandler(rotationActions, rotationPolicy, syncPolicy, new SourceFileNameFormat(sourceType, fileNameFormat), stormConfig);
-      sourceHandlerMap.put(sourceType, ret);
-    }
-    return ret;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
deleted file mode 100644
index 1c345b4..0000000
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.writer.hdfs;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Logger;
-import org.apache.storm.hdfs.common.rotation.RotationAction;
-
-import java.io.IOException;
-
-public class SourceAwareMoveAction implements RotationAction{
-  private static final Logger LOG = Logger.getLogger(SourceHandler.class);
-  private String destination;
-
-  public SourceAwareMoveAction toDestination(String destDir){
-    destination = destDir;
-    return this;
-  }
-
-  private static String getSource(Path filePath) {
-    return filePath.getParent().getName();
-  }
-
-  @Override
-  public void execute(FileSystem fileSystem, Path filePath) throws IOException {
-    Path destPath = new Path(new Path(destination, getSource(filePath)), filePath.getName());
-    LOG.info("Moving file " + filePath + " to " + destPath);
-    boolean success = fileSystem.rename(filePath, destPath);
-    return;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java
deleted file mode 100644
index ae0242d..0000000
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.writer.hdfs;
-
-import backtype.storm.task.TopologyContext;
-import org.apache.storm.hdfs.bolt.format.FileNameFormat;
-
-import java.util.Map;
-
-public class SourceFileNameFormat implements FileNameFormat {
-  FileNameFormat delegate;
-  String sourceType;
-  public SourceFileNameFormat(String sourceType, FileNameFormat delegate) {
-    this.delegate = delegate;
-    this.sourceType = sourceType;
-  }
-
-  @Override
-  public void prepare(Map map, TopologyContext topologyContext) {
-    this.delegate.prepare(map, topologyContext);
-  }
-
-  @Override
-  public String getName(long l, long l1) {
-    return delegate.getName(l, l1);
-  }
-
-  @Override
-  public String getPath() {
-    return delegate.getPath() + "/" + sourceType;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
deleted file mode 100644
index 0225137..0000000
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.writer.hdfs;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
-import org.apache.hadoop.hdfs.util.MD5FileUtils;
-import org.apache.hadoop.io.MD5Hash;
-import org.apache.log4j.Logger;
-import org.apache.storm.hdfs.bolt.format.FileNameFormat;
-import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
-import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy;
-import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
-import org.apache.storm.hdfs.common.rotation.RotationAction;
-import org.apache.storm.hdfs.common.security.HdfsSecurityUtil;
-import org.json.simple.JSONObject;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.*;
-
-public class SourceHandler {
-  private static final Logger LOG = Logger.getLogger(SourceHandler.class);
-  List<RotationAction> rotationActions = new ArrayList<>();
-  FileRotationPolicy rotationPolicy;
-  SyncPolicy syncPolicy;
-  FileNameFormat fileNameFormat;
-  private long offset = 0;
-  private int rotation = 0;
-  private transient FSDataOutputStream out;
-  private transient Object writeLock;
-  protected transient Timer rotationTimer; // only used for TimedRotationPolicy
-  protected transient FileSystem fs;
-  protected transient Path currentFile;
-  public SourceHandler(List<RotationAction> rotationActions
-                      , FileRotationPolicy rotationPolicy
-                      , SyncPolicy syncPolicy
-                      , FileNameFormat fileNameFormat
-                      , Map config
-                      ) throws IOException {
-    this.rotationActions = rotationActions;
-    this.rotationPolicy = rotationPolicy;
-    this.syncPolicy = syncPolicy;
-    this.fileNameFormat = fileNameFormat;
-    initialize(config);
-  }
-
-  public void handle(List<JSONObject> messages) throws Exception{
-
-    for(JSONObject message : messages) {
-      byte[] bytes = (message.toJSONString() + "\n").getBytes();
-      synchronized (this.writeLock) {
-        out.write(bytes);
-        this.offset += bytes.length;
-
-        if (this.syncPolicy.mark(null, this.offset)) {
-          if (this.out instanceof HdfsDataOutputStream) {
-            ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
-          } else {
-            this.out.hsync();
-          }
-          this.syncPolicy.reset();
-        }
-      }
-
-      if (this.rotationPolicy.mark(null, this.offset)) {
-        rotateOutputFile(); // synchronized
-        this.offset = 0;
-        this.rotationPolicy.reset();
-      }
-    }
-  }
-
-  private void initialize(Map config) throws IOException {
-    this.writeLock = new Object();
-    Configuration hdfsConfig = new Configuration();
-    this.fs = FileSystem.get(new Configuration());
-    HdfsSecurityUtil.login(config, hdfsConfig);
-    this.currentFile = createOutputFile();
-    if(this.rotationPolicy instanceof TimedRotationPolicy){
-      long interval = ((TimedRotationPolicy)this.rotationPolicy).getInterval();
-      this.rotationTimer = new Timer(true);
-      TimerTask task = new TimerTask() {
-        @Override
-        public void run() {
-          try {
-            rotateOutputFile();
-          } catch(IOException e){
-            LOG.warn("IOException during scheduled file rotation.", e);
-          }
-        }
-      };
-      this.rotationTimer.scheduleAtFixedRate(task, interval, interval);
-    }
-  }
-
-  protected void rotateOutputFile() throws IOException {
-    LOG.info("Rotating output file...");
-    long start = System.currentTimeMillis();
-    synchronized (this.writeLock) {
-      closeOutputFile();
-      this.rotation++;
-
-      Path newFile = createOutputFile();
-      LOG.info("Performing " +  this.rotationActions.size() + " file rotation actions." );
-      for (RotationAction action : this.rotationActions) {
-        action.execute(this.fs, this.currentFile);
-      }
-      this.currentFile = newFile;
-    }
-    long time = System.currentTimeMillis() - start;
-    LOG.info("File rotation took " + time + " ms.");
-  }
-
-  private Path createOutputFile() throws IOException {
-    Path path = new Path(this.fileNameFormat.getPath(), this.fileNameFormat.getName(this.rotation, System.currentTimeMillis()));
-    if(fs.getScheme().equals("file")) {
-      //in the situation where we're running this in a local filesystem, flushing doesn't work.
-      fs.mkdirs(path.getParent());
-      this.out = new FSDataOutputStream(new FileOutputStream(path.toString()), null);
-    }
-    else {
-      this.out = this.fs.create(path);
-    }
-    return path;
-  }
-
-  private void closeOutputFile() throws IOException {
-    this.out.close();
-  }
-
-
-  public void close() {
-    try {
-      closeOutputFile();
-    } catch (IOException e) {
-      throw new RuntimeException("Unable to close output file.", e);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/scripts/start_enrichment_topology.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/scripts/start_enrichment_topology.sh b/metron-platform/metron-enrichment/src/main/scripts/start_enrichment_topology.sh
new file mode 100755
index 0000000..6824b87
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/scripts/start_enrichment_topology.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+# 
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+METRON_VERSION=${project.version}
+METRON_HOME=/usr/metron/$METRON_VERSION
+TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
+storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/enrichment/remote.yaml --filter $METRON_HOME/config/enrichment.properties

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
index 04415e1..c42c284 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
@@ -21,11 +21,10 @@ import backtype.storm.tuple.Tuple;
 import backtype.storm.tuple.Values;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.EnrichmentConfigurations;
 import org.apache.metron.common.configuration.writer.WriterConfiguration;
 import org.apache.metron.test.bolt.BaseEnrichmentBoltTest;
-import org.apache.metron.common.configuration.Configurations;
 import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.writer.bolt.BulkMessageWriterBolt;
 import org.hamcrest.Description;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
new file mode 100644
index 0000000..6b12264
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
@@ -0,0 +1,426 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.integration;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.google.common.base.*;
+
+import com.google.common.collect.Iterables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.metron.TestConstants;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.converter.EnrichmentHelper;
+import org.apache.metron.integration.*;
+import org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
+import org.apache.metron.integration.utils.TestUtils;
+import org.apache.metron.test.utils.UnitTestHelper;
+import org.apache.metron.integration.components.FluxTopologyComponent;
+import org.apache.metron.integration.components.KafkaWithZKComponent;
+import org.apache.metron.enrichment.integration.mock.MockGeoAdapter;
+import org.apache.metron.test.mock.MockHTable;
+import org.apache.metron.enrichment.lookup.LookupKV;
+
+import org.apache.metron.enrichment.integration.utils.SampleUtil;
+import org.apache.metron.common.utils.JSONUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+public class EnrichmentIntegrationTest extends BaseIntegrationTest {
+
+  private static final String SRC_IP = "ip_src_addr";
+  private static final String DST_IP = "ip_dst_addr";
+  private static final String MALICIOUS_IP_TYPE = "malicious_ip";
+  private static final String PLAYFUL_CLASSIFICATION_TYPE = "playful_classification";
+  private static final Map<String, Object> PLAYFUL_ENRICHMENT = new HashMap<String, Object>() {{
+    put("orientation", "north");
+  }};
+  protected String testSensorType = "test";
+  protected String hdfsDir = "target/enrichmentIntegrationTest/hdfs";
+  protected String fluxPath = "../metron-enrichment/src/main/flux/enrichment/test.yaml";
+  protected String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
+  private String sampleIndexedPath = TestConstants.SAMPLE_DATA_INDEXED_PATH + "TestIndexed";
+
+  public static class Provider implements TableProvider, Serializable {
+    MockHTable.Provider  provider = new MockHTable.Provider();
+    @Override
+    public HTableInterface getTable(Configuration config, String tableName) throws IOException {
+      return provider.getTable(config, tableName);
+    }
+  }
+
+
+
+
+
+
+  @Test
+  public void test() throws Exception {
+    final EnrichmentConfigurations configurations = SampleUtil.getSampleEnrichmentConfigs();
+    final String dateFormat = "yyyy.MM.dd.HH";
+    final List<byte[]> inputMessages = TestUtils.readSampleData(sampleParsedPath);
+    final String cf = "cf";
+    final String trackerHBaseTableName = "tracker";
+    final String threatIntelTableName = "threat_intel";
+    final String enrichmentsTableName = "enrichments";
+    final Properties topologyProperties = new Properties() {{
+      setProperty("org.apache.metron.enrichment.host.known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"},\n" +
+              "{\"ip\":\"10.1.128.237\", \"local\":\"UNKNOWN\", \"type\":\"unknown\", \"asset_value\" : \"important\"},\n" +
+              "{\"ip\":\"10.60.10.254\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"},\n" +
+              "{\"ip\":\"10.0.2.15\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}]");
+      setProperty("hbase.provider.impl","" + Provider.class.getName());
+      setProperty("threat.intel.tracker.table", trackerHBaseTableName);
+      setProperty("threat.intel.tracker.cf", cf);
+      setProperty("threat.intel.simple.hbase.table", threatIntelTableName);
+      setProperty("threat.intel.simple.hbase.cf", cf);
+      setProperty("enrichment.simple.hbase.table", enrichmentsTableName);
+      setProperty("enrichment.simple.hbase.cf", cf);
+      setProperty("enrichment.output.topic", Constants.INDEXING_TOPIC);
+    }};
+    final KafkaWithZKComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaWithZKComponent.Topic>() {{
+      add(new KafkaWithZKComponent.Topic(Constants.ENRICHMENT_TOPIC, 1));
+      add(new KafkaWithZKComponent.Topic(Constants.INDEXING_TOPIC, 1));
+    }});
+
+    ConfigUploadComponent configUploadComponent = new ConfigUploadComponent()
+            .withTopologyProperties(topologyProperties)
+            .withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH)
+            .withEnrichmentConfigsPath(TestConstants.SAMPLE_CONFIG_PATH);
+
+    //create MockHBaseTables
+    final MockHTable trackerTable = (MockHTable)MockHTable.Provider.addToCache(trackerHBaseTableName, cf);
+    final MockHTable threatIntelTable = (MockHTable)MockHTable.Provider.addToCache(threatIntelTableName, cf);
+    EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>(){{
+      add(new LookupKV<>(new EnrichmentKey(MALICIOUS_IP_TYPE, "10.0.2.3"), new EnrichmentValue(new HashMap<>())));
+    }});
+    final MockHTable enrichmentTable = (MockHTable)MockHTable.Provider.addToCache(enrichmentsTableName, cf);
+    EnrichmentHelper.INSTANCE.load(enrichmentTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>(){{
+      add(new LookupKV<>(new EnrichmentKey(PLAYFUL_CLASSIFICATION_TYPE, "10.0.2.3")
+                        , new EnrichmentValue(PLAYFUL_ENRICHMENT )
+                        )
+         );
+    }});
+    FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder()
+            .withTopologyLocation(new File(fluxPath))
+            .withTopologyName("test")
+            .withTopologyProperties(topologyProperties)
+            .build();
+
+
+    UnitTestHelper.verboseLogging();
+    ComponentRunner runner = new ComponentRunner.Builder()
+            .withComponent("kafka", kafkaComponent)
+            .withComponent("config", configUploadComponent)
+            .withComponent("storm", fluxComponent)
+            .withMillisecondsBetweenAttempts(15000)
+            .withNumRetries(10)
+            .build();
+    runner.start();
+
+    try {
+      fluxComponent.submitTopology();
+
+      kafkaComponent.writeMessages(Constants.ENRICHMENT_TOPIC, inputMessages);
+      List<Map<String, Object>> docs = runner.process(getProcessor(inputMessages));
+      Assert.assertEquals(inputMessages.size(), docs.size());
+      List<Map<String, Object>> cleanedDocs = docs;
+      validateAll(cleanedDocs);
+    }
+    finally {
+      runner.stop();
+    }
+  }
+
+
+  public static void validateAll(List<Map<String, Object>> docs) {
+
+    for (Map<String, Object> doc : docs) {
+      baseValidation(doc);
+      hostEnrichmentValidation(doc);
+      geoEnrichmentValidation(doc);
+      threatIntelValidation(doc);
+      simpleEnrichmentValidation(doc);
+    }
+  }
+
+  public static void baseValidation(Map<String, Object> jsonDoc) {
+    assertEnrichmentsExists("threatintels.", setOf("hbaseThreatIntel"), jsonDoc.keySet());
+    assertEnrichmentsExists("enrichments.", setOf("geo", "host", "hbaseEnrichment" ), jsonDoc.keySet());
+    for(Map.Entry<String, Object> kv : jsonDoc.entrySet()) {
+      //ensure no values are empty.
+      Assert.assertTrue(kv.getValue().toString().length() > 0);
+    }
+    //ensure we always have a source ip and destination ip
+    Assert.assertNotNull(jsonDoc.get(SRC_IP));
+    Assert.assertNotNull(jsonDoc.get(DST_IP));
+  }
+
+  private static class EvaluationPayload {
+    Map<String, Object> indexedDoc;
+    String key;
+    public EvaluationPayload(Map<String, Object> indexedDoc, String key) {
+      this.indexedDoc = indexedDoc;
+      this.key = key;
+    }
+  }
+
+  private static enum HostEnrichments implements Predicate<EvaluationPayload>{
+
+    LOCAL_LOCATION(new Predicate<EvaluationPayload>() {
+
+      @Override
+      public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+
+        return evaluationPayload.indexedDoc.getOrDefault("enrichments.host." + evaluationPayload.key + ".known_info.local","").equals("YES");
+
+      }
+    })
+
+    ,UNKNOWN_LOCATION(new Predicate<EvaluationPayload>() {
+
+      @Override
+      public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+        return evaluationPayload.indexedDoc.getOrDefault("enrichments.host." + evaluationPayload.key + ".known_info.local","").equals("UNKNOWN");
+      }
+    })
+    ,IMPORTANT(new Predicate<EvaluationPayload>() {
+      @Override
+      public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+        return evaluationPayload.indexedDoc.getOrDefault("enrichments.host." + evaluationPayload.key + ".known_info.asset_value","").equals("important");
+      }
+    })
+    ,PRINTER_TYPE(new Predicate<EvaluationPayload>() {
+      @Override
+      public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+        return evaluationPayload.indexedDoc.getOrDefault("enrichments.host." + evaluationPayload.key + ".known_info.type","").equals("printer");
+      }
+    })
+    ,WEBSERVER_TYPE(new Predicate<EvaluationPayload>() {
+      @Override
+      public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+        return evaluationPayload.indexedDoc.getOrDefault("enrichments.host." + evaluationPayload.key + ".known_info.type","").equals("webserver");
+      }
+    })
+    ,UNKNOWN_TYPE(new Predicate<EvaluationPayload>() {
+      @Override
+      public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+        return evaluationPayload.indexedDoc.getOrDefault("enrichments.host." + evaluationPayload.key + ".known_info.type","").equals("unknown");
+      }
+    })
+    ;
+
+    Predicate<EvaluationPayload> _predicate;
+    HostEnrichments(Predicate<EvaluationPayload> predicate) {
+      this._predicate = predicate;
+    }
+
+    public boolean apply(EvaluationPayload payload) {
+      return _predicate.apply(payload);
+    }
+
+  }
+
+  private static void assertEnrichmentsExists(String topLevel, Set<String> expectedEnrichments, Set<String> keys) {
+    for(String key : keys) {
+      if(key.startsWith(topLevel)) {
+        String secondLevel = Iterables.get(Splitter.on(".").split(key), 1);
+        String message = "Found an enrichment/threat intel (" + secondLevel + ") that I didn't expect (expected enrichments :"
+                       + Joiner.on(",").join(expectedEnrichments) + "), but it was not there.  If you've created a new"
+                       + " enrichment, then please add a validation method to this unit test.  Otherwise, it's a solid error"
+                       + " and should be investigated.";
+        Assert.assertTrue( message, expectedEnrichments.contains(secondLevel));
+      }
+    }
+  }
+  private static void simpleEnrichmentValidation(Map<String, Object> indexedDoc) {
+    if(indexedDoc.getOrDefault(SRC_IP,"").equals("10.0.2.3")
+            || indexedDoc.getOrDefault(DST_IP,"").equals("10.0.2.3")
+            ) {
+      Assert.assertTrue(keyPatternExists("enrichments.hbaseEnrichment", indexedDoc));
+      if(indexedDoc.getOrDefault(SRC_IP,"").equals("10.0.2.3")) {
+        Assert.assertEquals(indexedDoc.get("enrichments.hbaseEnrichment." + SRC_IP + "." + PLAYFUL_CLASSIFICATION_TYPE+ ".orientation")
+                , PLAYFUL_ENRICHMENT.get("orientation")
+        );
+      }
+      else if(indexedDoc.getOrDefault(DST_IP,"").equals("10.0.2.3")) {
+        Assert.assertEquals( indexedDoc.get("enrichments.hbaseEnrichment." + DST_IP + "." + PLAYFUL_CLASSIFICATION_TYPE + ".orientation")
+                , PLAYFUL_ENRICHMENT.get("orientation")
+        );
+      }
+    }
+
+  }
+  private static void threatIntelValidation(Map<String, Object> indexedDoc) {
+    if(indexedDoc.getOrDefault(SRC_IP,"").equals("10.0.2.3")
+    || indexedDoc.getOrDefault(DST_IP,"").equals("10.0.2.3")
+            ) {
+      //if we have any threat intel messages, we want to tag is_alert to true
+      Assert.assertTrue(keyPatternExists("threatintels.", indexedDoc));
+      Assert.assertTrue(indexedDoc.containsKey("threat.triage.level"));
+      Assert.assertEquals(indexedDoc.getOrDefault("is_alert",""), "true");
+      Assert.assertEquals((double)indexedDoc.get("threat.triage.level"), 10d, 1e-7);
+    }
+    else {
+      //For YAF this is the case, but if we do snort later on, this will be invalid.
+      Assert.assertNull(indexedDoc.get("is_alert"));
+      Assert.assertFalse(keyPatternExists("threatintels.", indexedDoc));
+    }
+    //ip threat intels
+    if(keyPatternExists("threatintels.hbaseThreatIntel.", indexedDoc)) {
+      if(indexedDoc.getOrDefault(SRC_IP,"").equals("10.0.2.3")) {
+        Assert.assertEquals(indexedDoc.get("threatintels.hbaseThreatIntel." + SRC_IP + "." + MALICIOUS_IP_TYPE), "alert");
+      }
+      else if(indexedDoc.getOrDefault(DST_IP,"").equals("10.0.2.3")) {
+        Assert.assertEquals(indexedDoc.get("threatintels.hbaseThreatIntel." + DST_IP + "." + MALICIOUS_IP_TYPE), "alert");
+      }
+      else {
+        Assert.fail("There was a threat intels that I did not expect: " + indexedDoc);
+      }
+    }
+
+  }
+
+  private static void geoEnrichmentValidation(Map<String, Object> indexedDoc) {
+    //should have geo enrichment on every message due to mock geo adapter
+    Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".location_point"), MockGeoAdapter.DEFAULT_LOCATION_POINT);
+    Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP +".location_point"), MockGeoAdapter.DEFAULT_LOCATION_POINT);
+    Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".longitude"), MockGeoAdapter.DEFAULT_LONGITUDE);
+    Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".longitude"), MockGeoAdapter.DEFAULT_LONGITUDE);
+    Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".city"), MockGeoAdapter.DEFAULT_CITY);
+    Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".city"), MockGeoAdapter.DEFAULT_CITY);
+    Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".latitude"), MockGeoAdapter.DEFAULT_LATITUDE);
+    Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".latitude"), MockGeoAdapter.DEFAULT_LATITUDE);
+    Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".country"), MockGeoAdapter.DEFAULT_COUNTRY);
+    Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".country"), MockGeoAdapter.DEFAULT_COUNTRY);
+    Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".dmaCode"), MockGeoAdapter.DEFAULT_DMACODE);
+    Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".dmaCode"), MockGeoAdapter.DEFAULT_DMACODE);
+    Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".postalCode"), MockGeoAdapter.DEFAULT_POSTAL_CODE);
+    Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".postalCode"), MockGeoAdapter.DEFAULT_POSTAL_CODE);
+  }
+
+  private static void hostEnrichmentValidation(Map<String, Object> indexedDoc) {
+    boolean enriched = false;
+    //important local printers
+    {
+      Set<String> ips = setOf("10.0.2.15", "10.60.10.254");
+      if (ips.contains(indexedDoc.get(SRC_IP))) {
+        //this is a local, important, printer
+        Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
+                ,HostEnrichments.IMPORTANT
+                ,HostEnrichments.PRINTER_TYPE
+                ).apply(new EvaluationPayload(indexedDoc, SRC_IP))
+        );
+        enriched = true;
+      }
+      if (ips.contains(indexedDoc.get(DST_IP))) {
+        Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
+                ,HostEnrichments.IMPORTANT
+                ,HostEnrichments.PRINTER_TYPE
+                ).apply(new EvaluationPayload(indexedDoc, DST_IP))
+        );
+        enriched = true;
+      }
+    }
+    //important local webservers
+    {
+      Set<String> ips = setOf("10.1.128.236");
+      if (ips.contains(indexedDoc.get(SRC_IP))) {
+        //this is a local, important, printer
+        Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
+                ,HostEnrichments.IMPORTANT
+                ,HostEnrichments.WEBSERVER_TYPE
+                ).apply(new EvaluationPayload(indexedDoc, SRC_IP))
+        );
+        enriched = true;
+      }
+      if (ips.contains(indexedDoc.get(DST_IP))) {
+        Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
+                ,HostEnrichments.IMPORTANT
+                ,HostEnrichments.WEBSERVER_TYPE
+                ).apply(new EvaluationPayload(indexedDoc, DST_IP))
+        );
+        enriched = true;
+      }
+    }
+    if(!enriched) {
+      Assert.assertFalse(keyPatternExists("enrichments.host", indexedDoc));
+    }
+  }
+
+
+  private static boolean keyPatternExists(String pattern, Map<String, Object> indexedObj) {
+    for(String k : indexedObj.keySet()) {
+      if(k.startsWith(pattern)) {
+        return true;
+      }
+    }
+    return false;
+  }
+  private static Set<String> setOf(String... items) {
+    Set<String> ret = new HashSet<>();
+    for(String item : items) {
+      ret.add(item);
+    }
+    return ret;
+  }
+
+  public Processor<List<Map<String, Object>>> getProcessor(List<byte[]> inputMessages) {
+    return new Processor<List<Map<String, Object>>>() {
+      List<Map<String, Object>> docs = null;
+
+      public ReadinessState process(ComponentRunner runner) {
+        KafkaWithZKComponent kafkaComponent = runner.getComponent("kafka", KafkaWithZKComponent.class);
+        List<byte[]> messages = kafkaComponent.readMessages(Constants.INDEXING_TOPIC);
+        if (messages.size() == inputMessages.size()) {
+          docs = new ArrayList<>();
+          for(byte[] message : messages) {
+            try {
+              docs.add(JSONUtils.INSTANCE.load(new String(message), new TypeReference<Map<String, Object>>() {}));
+            } catch (IOException e) {
+              throw new IllegalStateException(e.getMessage(), e);
+            }
+          }
+          return ReadinessState.READY;
+        } else {
+          return ReadinessState.NOT_READY;
+        }
+      }
+
+      public List<Map<String, Object>> getResult() {
+        return docs;
+      }
+    };
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java
new file mode 100644
index 0000000..5cc14ac
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.integration.components;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.integration.InMemoryComponent;
+import org.apache.metron.integration.UnableToStartException;
+import org.apache.metron.integration.components.KafkaWithZKComponent;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+
+public class ConfigUploadComponent implements InMemoryComponent {
+
+  private Properties topologyProperties;
+  private String globalConfigPath;
+  private String parserConfigsPath;
+  private String enrichmentConfigsPath;
+  private Optional<String> globalConfig = Optional.empty();
+  private Map<String, SensorParserConfig> parserSensorConfigs = new HashMap<>();
+  public ConfigUploadComponent withTopologyProperties(Properties topologyProperties) {
+    this.topologyProperties = topologyProperties;
+    return this;
+  }
+
+  public ConfigUploadComponent withGlobalConfigsPath(String globalConfigPath) {
+    this.globalConfigPath = globalConfigPath;
+    return this;
+  }
+
+  public ConfigUploadComponent withParserConfigsPath(String parserConfigsPath) {
+    this.parserConfigsPath = parserConfigsPath;
+    return this;
+  }
+  public ConfigUploadComponent withEnrichmentConfigsPath(String enrichmentConfigsPath) {
+    this.enrichmentConfigsPath = enrichmentConfigsPath;
+    return this;
+  }
+
+  public ConfigUploadComponent withParserSensorConfig(String name, SensorParserConfig config) {
+    parserSensorConfigs.put(name, config);
+    return this;
+  }
+
+  public ConfigUploadComponent withGlobalConfig(String globalConfig) {
+    this.globalConfig = Optional.ofNullable(globalConfig);
+    return this;
+  }
+
+  @Override
+  public void start() throws UnableToStartException {
+    try {
+      if(globalConfigPath != null) {
+        ConfigurationsUtils.uploadConfigsToZookeeper(globalConfigPath
+                , parserConfigsPath
+                , enrichmentConfigsPath
+                , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
+        );
+      }
+      for(Map.Entry<String, SensorParserConfig> kv : parserSensorConfigs.entrySet()) {
+        ConfigurationsUtils.writeSensorParserConfigToZookeeper( kv.getKey()
+                                                              , kv.getValue()
+                                                              , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
+                                                              );
+      }
+      if(globalConfig.isPresent()) {
+        ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig.get().getBytes()
+                                                        , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
+                                                        );
+      }
+
+    } catch (Exception e) {
+      throw new UnableToStartException(e.getMessage(), e);
+    }
+  }
+
+  public SensorParserConfig getSensorParserConfig(String sensorType) {
+    SensorParserConfig sensorParserConfig = new SensorParserConfig();
+    CuratorFramework client = ConfigurationsUtils.getClient(topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY));
+    client.start();
+    try {
+      sensorParserConfig = ConfigurationsUtils.readSensorParserConfigFromZookeeper(sensorType, client);
+    } catch (Exception e) {
+      e.printStackTrace();
+    } finally {
+      client.close();
+    }
+    return sensorParserConfig;
+  }
+
+  @Override
+  public void stop() {
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockGeoAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockGeoAdapter.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockGeoAdapter.java
new file mode 100644
index 0000000..14a4493
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockGeoAdapter.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.integration.mock;
+
+import com.google.common.base.Joiner;
+import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
+import org.json.simple.JSONObject;
+
+import java.io.Serializable;
+
+public class MockGeoAdapter implements EnrichmentAdapter<CacheKey>,
+        Serializable {
+
+  public static final String DEFAULT_LOC_ID = "1";
+  public static final String DEFAULT_COUNTRY = "test country";
+  public static final String DEFAULT_CITY = "test city";
+  public static final String DEFAULT_POSTAL_CODE = "test postalCode";
+  public static final String DEFAULT_LATITUDE = "test latitude";
+  public static final String DEFAULT_LONGITUDE = "test longitude";
+  public static final String DEFAULT_DMACODE= "test dmaCode";
+  public static final String DEFAULT_LOCATION_POINT= Joiner.on(',').join(DEFAULT_LATITUDE,DEFAULT_LONGITUDE);
+
+  @Override
+  public void logAccess(CacheKey value) {
+
+  }
+
+  public JSONObject enrich(CacheKey cache ) {
+    JSONObject enriched = new JSONObject();
+    enriched.put("locID", DEFAULT_LOC_ID);
+    enriched.put("country", DEFAULT_COUNTRY);
+    enriched.put("city", DEFAULT_CITY);
+    enriched.put("postalCode", DEFAULT_POSTAL_CODE);
+    enriched.put("latitude", DEFAULT_LATITUDE);
+    enriched.put("longitude", DEFAULT_LONGITUDE);
+    enriched.put("dmaCode", DEFAULT_DMACODE);
+    enriched.put("location_point", DEFAULT_LOCATION_POINT);
+    return enriched;
+  }
+
+  public boolean initializeAdapter() {
+    return true;
+  }
+
+  public void cleanup() {
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockHBaseConnector.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockHBaseConnector.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockHBaseConnector.java
new file mode 100644
index 0000000..3b16b6a
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockHBaseConnector.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.integration.mock;
+
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
+import org.apache.metron.hbase.Connector;
+import org.apache.metron.hbase.TupleTableConfig;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class MockHBaseConnector extends Connector {
+    static List<Put> puts = Collections.synchronizedList(new ArrayList<Put>());
+    public MockHBaseConnector(TupleTableConfig conf, String _quorum, String _port) throws IOException {
+        super(conf, _quorum, _port);
+    }
+
+    @Override
+    public void put(Put put) throws InterruptedIOException, RetriesExhaustedWithDetailsException {
+        puts.add(put);
+    }
+
+    @Override
+    public void close() {
+
+    }
+    public static void clear() {
+        puts.clear();
+    }
+    public static List<Put> getPuts() {
+        return puts;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockTableProvider.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockTableProvider.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockTableProvider.java
new file mode 100644
index 0000000..ac2904a
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockTableProvider.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.enrichment.integration.mock;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.test.mock.MockHTable;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+public class MockTableProvider implements TableProvider, Serializable {
+  static MockHTable.Provider provider = new MockHTable.Provider();
+  @Override
+  public HTableInterface getTable(Configuration config, String tableName) throws IOException {
+    return provider.getTable(config, tableName);
+  }
+  public static void addTable(String tableName, String... cf) {
+    provider.addToCache(tableName, cf);
+  }
+  public static MockHTable getTable(String tableName) {
+    try {
+      return (MockHTable) provider.getTable(null, tableName);
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to get table: " + tableName);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java
new file mode 100644
index 0000000..40dcfe4
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.integration.utils;
+
+import org.apache.metron.TestConstants;
+import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.common.configuration.ParserConfigurations;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class SampleUtil {
+
+  public static Configurations getSampleConfigs() throws IOException {
+    Configurations configurations = new Configurations();
+    configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
+    return configurations;
+  }
+
+  public static ParserConfigurations getSampleParserConfigs() throws IOException {
+    ParserConfigurations configurations = new ParserConfigurations();
+    configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
+    Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH);
+    for(String sensorType: sensorParserConfigs.keySet()) {
+      configurations.updateSensorParserConfig(sensorType, sensorParserConfigs.get(sensorType));
+    }
+    return configurations;
+  }
+
+  public static EnrichmentConfigurations getSampleEnrichmentConfigs() throws IOException {
+    EnrichmentConfigurations configurations = new EnrichmentConfigurations();
+    configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
+    Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.SAMPLE_CONFIG_PATH);
+    for(String sensorType: sensorEnrichmentConfigs.keySet()) {
+      configurations.updateSensorEnrichmentConfig(sensorType, sensorEnrichmentConfigs.get(sensorType));
+    }
+    return configurations;
+  }
+
+}