You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/02/25 17:25:05 UTC

[1/6] incubator-metron git commit: METRON-50: Ingest threat intel data from Taxii feeds closes apache/incubator-metron#29

Repository: incubator-metron
Updated Branches:
  refs/heads/master a712fe69d -> 0e1055aaf


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/components/ElasticSearchComponent.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/components/ElasticSearchComponent.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/components/ElasticSearchComponent.java
deleted file mode 100644
index a7991c0..0000000
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/components/ElasticSearchComponent.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.util.integration.components;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.metron.integration.util.integration.InMemoryComponent;
-import org.apache.metron.integration.util.integration.UnableToStartException;
-import org.elasticsearch.ElasticsearchTimeoutException;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
-import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.ImmutableSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
-import org.elasticsearch.search.SearchHit;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class ElasticSearchComponent implements InMemoryComponent {
-
-    public static class Builder{
-        private int httpPort;
-        private File indexDir;
-        private Map<String, String> extraElasticSearchSettings = null;
-        public Builder withHttpPort(int httpPort) {
-            this.httpPort = httpPort;
-            return this;
-        }
-        public Builder withIndexDir(File indexDir) {
-            this.indexDir = indexDir;
-            return this;
-        }
-        public Builder withExtraElasticSearchSettings(Map<String, String> extraElasticSearchSettings) {
-            this.extraElasticSearchSettings = extraElasticSearchSettings;
-            return this;
-        }
-        public ElasticSearchComponent build() {
-            return new ElasticSearchComponent(httpPort, indexDir, extraElasticSearchSettings);
-        }
-    }
-
-    private Client client;
-    private Node node;
-    private int httpPort;
-    private File indexDir;
-    private Map<String, String> extraElasticSearchSettings;
-
-    public ElasticSearchComponent(int httpPort, File indexDir) {
-        this(httpPort, indexDir, null);
-    }
-    public ElasticSearchComponent(int httpPort, File indexDir, Map<String, String> extraElasticSearchSettings) {
-        this.httpPort = httpPort;
-        this.indexDir = indexDir;
-        this.extraElasticSearchSettings = extraElasticSearchSettings;
-    }
-    public Client getClient() {
-        return client;
-    }
-
-    private void cleanDir(File dir) throws IOException {
-        if(dir.exists()) {
-            FileUtils.deleteDirectory(dir);
-        }
-        dir.mkdirs();
-    }
-    public void start() throws UnableToStartException {
-        File logDir= new File(indexDir, "/logs");
-        File dataDir= new File(indexDir, "/data");
-        try {
-            cleanDir(logDir);
-            cleanDir(dataDir);
-
-        } catch (IOException e) {
-            throw new UnableToStartException("Unable to clean log or data directories", e);
-        }
-        ImmutableSettings.Builder immutableSettings = ImmutableSettings.settingsBuilder()
-                .put("node.http.enabled", true)
-                .put("http.port", httpPort)
-                .put("cluster.name", "metron")
-                .put("path.logs",logDir.getAbsolutePath())
-                .put("path.data",dataDir.getAbsolutePath())
-                .put("gateway.type", "none")
-                .put("index.store.type", "memory")
-                .put("index.number_of_shards", 1)
-                .put("node.mode", "network")
-                .put("index.number_of_replicas", 1);
-        if(extraElasticSearchSettings != null) {
-            immutableSettings = immutableSettings.put(extraElasticSearchSettings);
-        }
-        Settings settings = immutableSettings.build();
-
-        node = NodeBuilder.nodeBuilder().settings(settings).node();
-        node.start();
-        settings = ImmutableSettings.settingsBuilder()
-					.put("cluster.name", "metron").build();
-		client = new TransportClient(settings)
-					.addTransportAddress(new InetSocketTransportAddress("localhost",
-							9300));
-
-        waitForCluster(client, ClusterHealthStatus.YELLOW, new TimeValue(60000));
-    }
-
-    public static void waitForCluster(ElasticsearchClient client, ClusterHealthStatus status, TimeValue timeout) throws UnableToStartException {
-        try {
-            ClusterHealthResponse healthResponse =
-                    (ClusterHealthResponse)client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().waitForStatus(status).timeout(timeout)).actionGet();
-            if (healthResponse != null && healthResponse.isTimedOut()) {
-                throw new UnableToStartException("cluster state is " + healthResponse.getStatus().name()
-                        + " and not " + status.name()
-                        + ", from here on, everything will fail!");
-            }
-        } catch (ElasticsearchTimeoutException e) {
-            throw new UnableToStartException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
-        }
-    }
-
-    public List<Map<String, Object>> getAllIndexedDocs(String index) throws IOException {
-       return getAllIndexedDocs(index, "message");
-    }
-    public List<Map<String, Object>> getAllIndexedDocs(String index, String subMessage) throws IOException {
-        getClient().admin().indices().refresh(new RefreshRequest());
-        SearchResponse response = getClient().prepareSearch(index)
-                .setTypes("pcap_doc")
-                .setSource("message")
-                .setFrom(0)
-                .setSize(1000)
-                .execute().actionGet();
-        List<Map<String, Object>> ret = new ArrayList<Map<String, Object>>();
-        for (SearchHit hit : response.getHits()) {
-            Object o = null;
-            if(subMessage == null) {
-                o = hit.getSource();
-            }
-            else {
-                o = hit.getSource().get(subMessage);
-            }
-            ret.add((Map<String, Object>)(o));
-        }
-        return ret;
-    }
-    public boolean hasIndex(String indexName) {
-        Set<String> indices = getClient().admin()
-                                    .indices()
-                                    .stats(new IndicesStatsRequest())
-                                    .actionGet()
-                                    .getIndices()
-                                    .keySet();
-        return indices.contains(indexName);
-
-    }
-
-    public void stop() {
-        node.stop();
-        node = null;
-        client = null;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/components/FluxTopologyComponent.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/components/FluxTopologyComponent.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/components/FluxTopologyComponent.java
deleted file mode 100644
index 2cac4ee..0000000
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/components/FluxTopologyComponent.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.util.integration.components;
-
-import backtype.storm.Config;
-import backtype.storm.LocalCluster;
-import backtype.storm.generated.StormTopology;
-import org.apache.metron.integration.util.integration.InMemoryComponent;
-import org.apache.metron.integration.util.integration.UnableToStartException;
-import org.apache.storm.flux.FluxBuilder;
-import org.apache.storm.flux.model.ExecutionContext;
-import org.apache.storm.flux.model.TopologyDef;
-import org.apache.storm.flux.parser.FluxParser;
-import org.apache.thrift7.TException;
-import org.junit.Assert;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.util.Properties;
-
-public class FluxTopologyComponent implements InMemoryComponent {
-    LocalCluster stormCluster;
-    String topologyName;
-    File topologyLocation;
-    Properties topologyProperties;
-
-    public static class Builder {
-        String topologyName;
-        File topologyLocation;
-        Properties topologyProperties;
-        public Builder withTopologyName(String name) {
-            this.topologyName = name;
-            return this;
-        }
-        public Builder withTopologyLocation(File location) {
-            this.topologyLocation = location;
-            return this;
-        }
-        public Builder withTopologyProperties(Properties properties) {
-            this.topologyProperties = properties;
-            return this;
-        }
-
-        public FluxTopologyComponent build() {
-            return new FluxTopologyComponent(topologyName, topologyLocation, topologyProperties);
-        }
-    }
-
-    public FluxTopologyComponent(String topologyName, File topologyLocation, Properties topologyProperties) {
-        this.topologyName = topologyName;
-        this.topologyLocation = topologyLocation;
-        this.topologyProperties = topologyProperties;
-    }
-
-    public LocalCluster getStormCluster() {
-        return stormCluster;
-    }
-
-    public String getTopologyName() {
-        return topologyName;
-    }
-
-    public File getTopologyLocation() {
-        return topologyLocation;
-    }
-
-    public Properties getTopologyProperties() {
-        return topologyProperties;
-    }
-
-    public void start() throws UnableToStartException{
-        try {
-            stormCluster = new LocalCluster();
-        } catch (Exception e) {
-            throw new UnableToStartException("Unable to start flux topology: " + getTopologyLocation(), e);
-        }
-    }
-
-    public void stop() {
-        stormCluster.shutdown();
-    }
-
-    public void submitTopology() throws NoSuchMethodException, IOException, InstantiationException, TException, IllegalAccessException, InvocationTargetException, ClassNotFoundException {
-        startTopology(getTopologyName(), getTopologyLocation(), getTopologyProperties());
-    }
-    private void startTopology(String topologyName, File topologyLoc, Properties properties) throws IOException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, TException {
-        TopologyDef topologyDef = loadYaml(topologyName, topologyLoc, properties);
-        Config conf = FluxBuilder.buildConfig(topologyDef);
-        ExecutionContext context = new ExecutionContext(topologyDef, conf);
-        StormTopology topology = FluxBuilder.buildTopology(context);
-        Assert.assertNotNull(topology);
-        topology.validate();
-        stormCluster.submitTopology(topologyName, conf, topology);
-    }
-
-    private static TopologyDef loadYaml(String topologyName, File yamlFile, Properties properties) throws IOException {
-        File tmpFile = File.createTempFile(topologyName, "props");
-        tmpFile.deleteOnExit();
-        FileWriter propWriter = null;
-        try {
-            propWriter = new FileWriter(tmpFile);
-            properties.store(propWriter, topologyName + " properties");
-        }
-        finally {
-            if(propWriter != null) {
-                propWriter.close();
-                return FluxParser.parseFile(yamlFile.getAbsolutePath(), false, true, tmpFile.getAbsolutePath(), false);
-            }
-
-            return null;
-        }
-    }
-
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/mock/MockHTable.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/mock/MockHTable.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/mock/MockHTable.java
deleted file mode 100644
index cbc6d7d..0000000
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/mock/MockHTable.java
+++ /dev/null
@@ -1,673 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.util.mock;
-
-
-import com.google.protobuf.Descriptors;
-import com.google.protobuf.Message;
-import com.google.protobuf.Service;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.client.coprocessor.Batch;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.metron.hbase.TableProvider;
-
-import java.io.IOException;
-import java.util.*;
-
-/**
- * MockHTable.
- *
- * This implementation is a selected excerpt from https://gist.github.com/agaoglu/613217
- */
-public class MockHTable implements HTableInterface {
-
-    public static class Provider implements TableProvider {
-        private static Map<String, HTableInterface> _cache = new HashMap<>();
-        @Override
-        public HTableInterface getTable(Configuration config, String tableName) throws IOException {
-            return _cache.get(tableName);
-        }
-        public static HTableInterface getFromCache(String tableName) {
-            return _cache.get(tableName);
-        }
-        public static HTableInterface addToCache(String tableName, String... columnFamilies) {
-            MockHTable ret =  new MockHTable(tableName, columnFamilies);
-            _cache.put(tableName, ret);
-            return ret;
-        }
-
-        public static void clear() {
-            _cache.clear();
-        }
-    }
-
-    private final String tableName;
-    private final List<String> columnFamilies = new ArrayList<>();
-    private HColumnDescriptor[] descriptors;
-
-    private NavigableMap<byte[], NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>>> data
-            = new TreeMap<>(Bytes.BYTES_COMPARATOR);
-
-    private static List<KeyValue> toKeyValue(byte[] row, NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowdata, int maxVersions) {
-        return toKeyValue(row, rowdata, 0, Long.MAX_VALUE, maxVersions);
-    }
-
-    private static List<KeyValue> toKeyValue(byte[] row, NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowdata, long timestampStart, long timestampEnd, int maxVersions) {
-        List<KeyValue> ret = new ArrayList<KeyValue>();
-        for (byte[] family : rowdata.keySet())
-            for (byte[] qualifier : rowdata.get(family).keySet()) {
-                int versionsAdded = 0;
-                for (Map.Entry<Long, byte[]> tsToVal : rowdata.get(family).get(qualifier).descendingMap().entrySet()) {
-                    if (versionsAdded++ == maxVersions)
-                        break;
-                    Long timestamp = tsToVal.getKey();
-                    if (timestamp < timestampStart)
-                        continue;
-                    if (timestamp > timestampEnd)
-                        continue;
-                    byte[] value = tsToVal.getValue();
-                    ret.add(new KeyValue(row, family, qualifier, timestamp, value));
-                }
-            }
-        return ret;
-    }
-    public MockHTable(String tableName) {
-        this.tableName = tableName;
-    }
-
-    public MockHTable(String tableName, String... columnFamilies) {
-        this.tableName = tableName;
-        for(String cf : columnFamilies) {
-            addColumnFamily(cf);
-        }
-    }
-
-    public void addColumnFamily(String columnFamily) {
-        this.columnFamilies.add(columnFamily);
-        descriptors = new HColumnDescriptor[columnFamilies.size()];
-        int i = 0;
-        for(String cf : columnFamilies) {
-            descriptors[i++] = new HColumnDescriptor(cf);
-        }
-    }
-
-
-    @Override
-    public byte[] getTableName() {
-        return Bytes.toBytes(tableName);
-    }
-
-    @Override
-    public TableName getName() {
-        return TableName.valueOf(tableName);
-    }
-
-    @Override
-    public Configuration getConfiguration() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public HTableDescriptor getTableDescriptor() throws IOException {
-        HTableDescriptor ret = new HTableDescriptor(tableName);
-        for(HColumnDescriptor c : descriptors) {
-            ret.addFamily(c);
-        }
-        return ret;
-    }
-
-    @Override
-    public boolean exists(Get get) throws IOException {
-        if(get.getFamilyMap() == null || get.getFamilyMap().size() == 0) {
-            return data.containsKey(get.getRow());
-        } else {
-            byte[] row = get.getRow();
-            if(!data.containsKey(row)) {
-                return false;
-            }
-            for(byte[] family : get.getFamilyMap().keySet()) {
-                if(!data.get(row).containsKey(family)) {
-                    return false;
-                } else {
-                    return true;
-                }
-            }
-            return true;
-        }
-    }
-
-    /**
-     * Test for the existence of columns in the table, as specified by the Gets.
-     * <p/>
-     * <p/>
-     * This will return an array of booleans. Each value will be true if the related Get matches
-     * one or more keys, false if not.
-     * <p/>
-     * <p/>
-     * This is a server-side call so it prevents any data from being transferred to
-     * the client.
-     *
-     * @param gets the Gets
-     * @return Array of boolean.  True if the specified Get matches one or more keys, false if not.
-     * @throws IOException e
-     */
-    @Override
-    public boolean[] existsAll(List<Get> gets) throws IOException {
-        boolean[] ret = new boolean[gets.size()];
-        int i = 0;
-        for(boolean b : exists(gets)) {
-           ret[i++] = b;
-        }
-        return ret;
-    }
-
-    @Override
-    public Boolean[] exists(List<Get> list) throws IOException {
-        Boolean[] ret = new Boolean[list.size()];
-        int i = 0;
-        for(Get g : list) {
-           ret[i++] = exists(g);
-        }
-        return ret;
-    }
-
-    @Override
-    public void batch(List<? extends Row> list, Object[] objects) throws IOException, InterruptedException {
-        throw new UnsupportedOperationException();
-
-    }
-
-    /**
-     * @param actions
-     * @deprecated
-     */
-    @Override
-    public Object[] batch(List<? extends Row> actions) throws IOException, InterruptedException {
-        List<Result> results = new ArrayList<Result>();
-        for (Row r : actions) {
-            if (r instanceof Delete) {
-                delete((Delete) r);
-                continue;
-            }
-            if (r instanceof Put) {
-                put((Put) r);
-                continue;
-            }
-            if (r instanceof Get) {
-                results.add(get((Get) r));
-            }
-        }
-        return results.toArray();
-    }
-
-    @Override
-    public <R> void batchCallback(List<? extends Row> list, Object[] objects, Batch.Callback<R> callback) throws IOException, InterruptedException {
-        throw new UnsupportedOperationException();
-
-    }
-
-    /**
-     * @param list
-     * @param callback
-     * @deprecated
-     */
-    @Override
-    public <R> Object[] batchCallback(List<? extends Row> list, Batch.Callback<R> callback) throws IOException, InterruptedException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Result get(Get get) throws IOException {
-        if (!data.containsKey(get.getRow()))
-            return new Result();
-        byte[] row = get.getRow();
-        List<KeyValue> kvs = new ArrayList<KeyValue>();
-        if (!get.hasFamilies()) {
-            kvs = toKeyValue(row, data.get(row), get.getMaxVersions());
-        } else {
-            for (byte[] family : get.getFamilyMap().keySet()){
-                if (data.get(row).get(family) == null)
-                    continue;
-                NavigableSet<byte[]> qualifiers = get.getFamilyMap().get(family);
-                if (qualifiers == null || qualifiers.isEmpty())
-                    qualifiers = data.get(row).get(family).navigableKeySet();
-                for (byte[] qualifier : qualifiers){
-                    if (qualifier == null)
-                        qualifier = "".getBytes();
-                    if (!data.get(row).containsKey(family) ||
-                            !data.get(row).get(family).containsKey(qualifier) ||
-                            data.get(row).get(family).get(qualifier).isEmpty())
-                        continue;
-                    Map.Entry<Long, byte[]> timestampAndValue = data.get(row).get(family).get(qualifier).lastEntry();
-                    kvs.add(new KeyValue(row,family, qualifier, timestampAndValue.getKey(), timestampAndValue.getValue()));
-                }
-            }
-        }
-        Filter filter = get.getFilter();
-        if (filter != null) {
-            filter.reset();
-            List<KeyValue> nkvs = new ArrayList<KeyValue>(kvs.size());
-            for (KeyValue kv : kvs) {
-                if (filter.filterAllRemaining()) {
-                    break;
-                }
-                if (filter.filterRowKey(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength())) {
-                    continue;
-                }
-                if (filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE) {
-                    nkvs.add(kv);
-                }
-                // ignoring next key hint which is a optimization to reduce file system IO
-            }
-            if (filter.hasFilterRow()) {
-                filter.filterRow();
-            }
-            kvs = nkvs;
-        }
-
-        return new Result(kvs);
-    }
-
-    @Override
-    public Result[] get(List<Get> list) throws IOException {
-        Result[] ret = new Result[list.size()];
-        int i = 0;
-        for(Get g : list) {
-            ret[i++] = get(g);
-        }
-        return ret;
-    }
-
-    /**
-     * @param bytes
-     * @param bytes1
-     * @deprecated
-     */
-    @Override
-    public Result getRowOrBefore(byte[] bytes, byte[] bytes1) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public ResultScanner getScanner(Scan scan) throws IOException {
-        final List<Result> ret = new ArrayList<Result>();
-        byte[] st = scan.getStartRow();
-        byte[] sp = scan.getStopRow();
-        Filter filter = scan.getFilter();
-
-        for (byte[] row : data.keySet()){
-            // if row is equal to startRow emit it. When startRow (inclusive) and
-            // stopRow (exclusive) is the same, it should not be excluded which would
-            // happen w/o this control.
-            if (st != null && st.length > 0 &&
-                    Bytes.BYTES_COMPARATOR.compare(st, row) != 0) {
-                // if row is before startRow do not emit, pass to next row
-                if (st != null && st.length > 0 &&
-                        Bytes.BYTES_COMPARATOR.compare(st, row) > 0)
-                    continue;
-                // if row is equal to stopRow or after it do not emit, stop iteration
-                if (sp != null && sp.length > 0 &&
-                        Bytes.BYTES_COMPARATOR.compare(sp, row) <= 0)
-                    break;
-            }
-
-            List<KeyValue> kvs = null;
-            if (!scan.hasFamilies()) {
-                kvs = toKeyValue(row, data.get(row), scan.getTimeRange().getMin(), scan.getTimeRange().getMax(), scan.getMaxVersions());
-            } else {
-                kvs = new ArrayList<KeyValue>();
-                for (byte[] family : scan.getFamilyMap().keySet()){
-                    if (data.get(row).get(family) == null)
-                        continue;
-                    NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(family);
-                    if (qualifiers == null || qualifiers.isEmpty())
-                        qualifiers = data.get(row).get(family).navigableKeySet();
-                    for (byte[] qualifier : qualifiers){
-                        if (data.get(row).get(family).get(qualifier) == null)
-                            continue;
-                        for (Long timestamp : data.get(row).get(family).get(qualifier).descendingKeySet()){
-                            if (timestamp < scan.getTimeRange().getMin())
-                                continue;
-                            if (timestamp > scan.getTimeRange().getMax())
-                                continue;
-                            byte[] value = data.get(row).get(family).get(qualifier).get(timestamp);
-                            kvs.add(new KeyValue(row, family, qualifier, timestamp, value));
-                            if(kvs.size() == scan.getMaxVersions()) {
-                                break;
-                            }
-                        }
-                    }
-                }
-            }
-            if (filter != null) {
-                filter.reset();
-                List<KeyValue> nkvs = new ArrayList<KeyValue>(kvs.size());
-                for (KeyValue kv : kvs) {
-                    if (filter.filterAllRemaining()) {
-                        break;
-                    }
-                    if (filter.filterRowKey(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength())) {
-                        continue;
-                    }
-                    Filter.ReturnCode filterResult = filter.filterKeyValue(kv);
-                    if (filterResult == Filter.ReturnCode.INCLUDE) {
-                        nkvs.add(kv);
-                    } else if (filterResult == Filter.ReturnCode.NEXT_ROW) {
-                        break;
-                    }
-                    // ignoring next key hint which is a optimization to reduce file system IO
-                }
-                if (filter.hasFilterRow()) {
-                    filter.filterRow();
-                }
-                kvs = nkvs;
-            }
-            if (!kvs.isEmpty()) {
-                ret.add(new Result(kvs));
-            }
-        }
-
-        return new ResultScanner() {
-            private final Iterator<Result> iterator = ret.iterator();
-            public Iterator<Result> iterator() {
-                return iterator;
-            }
-            public Result[] next(int nbRows) throws IOException {
-                ArrayList<Result> resultSets = new ArrayList<Result>(nbRows);
-                for(int i = 0; i < nbRows; i++) {
-                    Result next = next();
-                    if (next != null) {
-                        resultSets.add(next);
-                    } else {
-                        break;
-                    }
-                }
-                return resultSets.toArray(new Result[resultSets.size()]);
-            }
-            public Result next() throws IOException {
-                try {
-                    return iterator().next();
-                } catch (NoSuchElementException e) {
-                    return null;
-                }
-            }
-            public void close() {}
-        };
-    }
-    @Override
-    public ResultScanner getScanner(byte[] family) throws IOException {
-        Scan scan = new Scan();
-        scan.addFamily(family);
-        return getScanner(scan);
-    }
-
-    @Override
-    public ResultScanner getScanner(byte[] family, byte[] qualifier)
-            throws IOException {
-        Scan scan = new Scan();
-        scan.addColumn(family, qualifier);
-        return getScanner(scan);
-    }
-
-    List<Put> putLog = new ArrayList<>();
-
-    public List<Put> getPutLog() {
-        return putLog;
-    }
-
-    @Override
-    public void put(Put put) throws IOException {
-        putLog.add(put);
-        byte[] row = put.getRow();
-        NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowData = forceFind(data, row, new TreeMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>>(Bytes.BYTES_COMPARATOR));
-        for (byte[] family : put.getFamilyMap().keySet()){
-            NavigableMap<byte[], NavigableMap<Long, byte[]>> familyData = forceFind(rowData, family, new TreeMap<byte[], NavigableMap<Long, byte[]>>(Bytes.BYTES_COMPARATOR));
-            for (KeyValue kv : put.getFamilyMap().get(family)){
-                kv.updateLatestStamp(Bytes.toBytes(System.currentTimeMillis()));
-                byte[] qualifier = kv.getQualifier();
-                NavigableMap<Long, byte[]> qualifierData = forceFind(familyData, qualifier, new TreeMap<Long, byte[]>());
-                qualifierData.put(kv.getTimestamp(), kv.getValue());
-            }
-        }
-    }
-
-    /**
-     * Helper method to find a key in a map. If key is not found, newObject is
-     * added to map and returned
-     *
-     * @param map
-     *          map to extract value from
-     * @param key
-     *          key to look for
-     * @param newObject
-     *          set key to this if not found
-     * @return found value or newObject if not found
-     */
-    private <K, V> V forceFind(NavigableMap<K, V> map, K key, V newObject){
-        V data = map.get(key);
-        if (data == null){
-            data = newObject;
-            map.put(key, data);
-        }
-        return data;
-    }
-
-    @Override
-    public void put(List<Put> puts) throws IOException {
-        for (Put put : puts)
-            put(put);
-    }
-
-    @Override
-    public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, byte[] bytes3, Put put) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
-    /**
-     * Atomically checks if a row/family/qualifier value matches the expected
-     * value. If it does, it adds the put.  If the passed value is null, the check
-     * is for the lack of column (ie: non-existance)
-     *
-     * @param row       to check
-     * @param family    column family to check
-     * @param qualifier column qualifier to check
-     * @param compareOp comparison operator to use
-     * @param value     the expected value
-     * @param put       data to put if check succeeds
-     * @return true if the new put was executed, false otherwise
-     * @throws IOException e
-     */
-    @Override
-    public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, CompareFilter.CompareOp compareOp, byte[] value, Put put) throws IOException {
-        return false;
-    }
-
-    @Override
-    public void delete(Delete delete) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void delete(List<Delete> list) throws IOException {
-        throw new UnsupportedOperationException();
-
-    }
-
-    @Override
-    public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, byte[] bytes3, Delete delete) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
-    /**
-     * Atomically checks if a row/family/qualifier value matches the expected
-     * value. If it does, it adds the delete.  If the passed value is null, the
-     * check is for the lack of column (ie: non-existance)
-     *
-     * @param row       to check
-     * @param family    column family to check
-     * @param qualifier column qualifier to check
-     * @param compareOp comparison operator to use
-     * @param value     the expected value
-     * @param delete    data to delete if check succeeds
-     * @return true if the new delete was executed, false otherwise
-     * @throws IOException e
-     */
-    @Override
-    public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, CompareFilter.CompareOp compareOp, byte[] value, Delete delete) throws IOException {
-        return false;
-    }
-
-    @Override
-    public void mutateRow(RowMutations rowMutations) throws IOException {
-        throw new UnsupportedOperationException();
-
-    }
-
-    @Override
-    public Result append(Append append) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Result increment(Increment increment) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public long incrementColumnValue(byte[] bytes, byte[] bytes1, byte[] bytes2, long l) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public long incrementColumnValue(byte[] bytes, byte[] bytes1, byte[] bytes2, long l, Durability durability) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
-    /**
-     * @param bytes
-     * @param bytes1
-     * @param bytes2
-     * @param l
-     * @param b
-     * @deprecated
-     */
-    @Override
-    public long incrementColumnValue(byte[] bytes, byte[] bytes1, byte[] bytes2, long l, boolean b) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public boolean isAutoFlush() {
-        return autoflush;
-    }
-
-    @Override
-    public void flushCommits() throws IOException {
-
-    }
-
-    @Override
-    public void close() throws IOException {
-
-    }
-
-    @Override
-    public CoprocessorRpcChannel coprocessorService(byte[] bytes) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> aClass, byte[] bytes, byte[] bytes1, Batch.Call<T, R> call) throws ServiceException, Throwable {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public <T extends Service, R> void coprocessorService(Class<T> aClass, byte[] bytes, byte[] bytes1, Batch.Call<T, R> call, Batch.Callback<R> callback) throws ServiceException, Throwable {
-        throw new UnsupportedOperationException();
-    }
-
-    boolean autoflush = true;
-
-    /**
-     * @param b
-     * @deprecated
-     */
-    @Override
-    public void setAutoFlush(boolean b) {
-        autoflush = b;
-    }
-
-    @Override
-    public void setAutoFlush(boolean b, boolean b1) {
-        autoflush = b;
-    }
-
-    @Override
-    public void setAutoFlushTo(boolean b) {
-        autoflush = b;
-    }
-
-    long writeBufferSize = 0;
-    @Override
-    public long getWriteBufferSize() {
-        return writeBufferSize;
-    }
-
-    @Override
-    public void setWriteBufferSize(long l) throws IOException {
-        writeBufferSize = l;
-    }
-
-    @Override
-    public <R extends Message> Map<byte[], R> batchCoprocessorService(Descriptors.MethodDescriptor methodDescriptor, Message message, byte[] bytes, byte[] bytes1, R r) throws ServiceException, Throwable {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public <R extends Message> void batchCoprocessorService(Descriptors.MethodDescriptor methodDescriptor, Message message, byte[] bytes, byte[] bytes1, R r, Batch.Callback<R> callback) throws ServiceException, Throwable {
-        throw new UnsupportedOperationException();
-    }
-
-    /**
-     * Atomically checks if a row/family/qualifier value matches the expected value.
-     * If it does, it performs the row mutations.  If the passed value is null, the check
-     * is for the lack of column (ie: non-existence)
-     *
-     * @param row       to check
-     * @param family    column family to check
-     * @param qualifier column qualifier to check
-     * @param compareOp the comparison operator
-     * @param value     the expected value
-     * @param mutation  mutations to perform if check succeeds
-     * @return true if the new put was executed, false otherwise
-     * @throws IOException e
-     */
-    @Override
-    public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, CompareFilter.CompareOp compareOp, byte[] value, RowMutations mutation) throws IOException {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/threatintel/ThreatIntelHelper.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/threatintel/ThreatIntelHelper.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/threatintel/ThreatIntelHelper.java
index 57d7902..1e64362 100644
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/threatintel/ThreatIntelHelper.java
+++ b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/threatintel/ThreatIntelHelper.java
@@ -19,17 +19,21 @@ package org.apache.metron.integration.util.threatintel;
 
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
+import org.apache.metron.reference.lookup.LookupKV;
 import org.apache.metron.threatintel.ThreatIntelResults;
-import org.apache.metron.threatintel.hbase.Converter;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelConverter;
 
 import java.io.IOException;
 
 public enum ThreatIntelHelper {
     INSTANCE;
+    ThreatIntelConverter converter = new ThreatIntelConverter();
 
-    public void load(HTableInterface table, String cf, Iterable<ThreatIntelResults> results, long ts) throws IOException {
-        for(ThreatIntelResults result : results) {
-            Put put = Converter.INSTANCE.toPut(cf, result.getKey(), result.getValue(), ts);
+    public void load(HTableInterface table, String cf, Iterable<LookupKV<ThreatIntelKey, ThreatIntelValue>> results) throws IOException {
+        for(LookupKV<ThreatIntelKey, ThreatIntelValue> result : results) {
+            Put put = converter.toPut(cf, result.getKey(), result.getValue());
             table.put(put);
         }
     }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/pom.xml b/metron-streaming/pom.xml
index 47ab2cb..827c8d0 100644
--- a/metron-streaming/pom.xml
+++ b/metron-streaming/pom.xml
@@ -69,6 +69,7 @@
 		<module>Metron-DataLoads</module>
 		<module>Metron-Topologies</module>
 		<module>Metron-Pcap_Service</module>
+		<module>Metron-Testing</module>
 	</modules>
 	<dependencies>
 		<dependency>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 992f328..a4c773d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,10 +45,12 @@
 						<exclude>**/.*</exclude>
 						<exclude>**/.*/**</exclude>
 						<exclude>**/*.seed</exclude>
+            <exclude>**/*.iml</exclude>
 						<exclude>**/ansible.cfg</exclude>
 						<exclude>site/**</exclude>
 						<exclude>metron-ui/lib/public/**</exclude>
 						<exclude>**/src/main/resources/patterns/**</exclude>
+						<exclude>**/src/test/resources/**</exclude>
 						<exclude>**/src/main/resources/SampleInput/**</exclude>
 						<exclude>**/dependency-reduced-pom.xml</exclude>
 					        <exclude>**/files/opensoc-ui</exclude>



[6/6] incubator-metron git commit: METRON-50: Ingest threat intel data from Taxii feeds closes apache/incubator-metron#29

Posted by ce...@apache.org.
METRON-50: Ingest threat intel data from Taxii feeds closes apache/incubator-metron#29


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/0e1055aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/0e1055aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/0e1055aa

Branch: refs/heads/master
Commit: 0e1055aafe78e428b0ad1fc085589a202cf14206
Parents: a712fe6
Author: cstella <ce...@gmail.com>
Authored: Thu Feb 25 11:24:56 2016 -0500
Committer: cstella <ce...@gmail.com>
Committed: Thu Feb 25 11:24:56 2016 -0500

----------------------------------------------------------------------
 NOTICE                                          |    3 +
 metron-streaming/Metron-Common/pom.xml          |    6 +
 .../hbase/converters/AbstractConverter.java     |   93 +
 .../metron/hbase/converters/HbaseConverter.java |   40 +
 .../threatintel/ThreatIntelConverter.java       |   39 +
 .../converters/threatintel/ThreatIntelKey.java  |   89 +
 .../threatintel/ThreatIntelValue.java           |  110 +
 .../metron/reference/lookup/LookupKV.java       |   65 +
 .../metron/reference/lookup/LookupKey.java      |    1 +
 .../metron/reference/lookup/LookupValue.java    |   28 +
 .../metron/threatintel/ThreatIntelKey.java      |   89 -
 .../metron/threatintel/ThreatIntelResults.java  |    2 +
 .../metron/threatintel/hbase/Converter.java     |   96 -
 .../threatintel/hbase/ThreatIntelLookup.java    |   24 +-
 metron-streaming/Metron-DataLoads/pom.xml       |  432 +--
 .../src/main/bash/threatintel_bulk_load.sh      |    2 +-
 .../src/main/bash/threatintel_bulk_prune.sh     |    2 +-
 .../src/main/bash/threatintel_taxii_load.sh     |   39 +
 .../dataloads/LeastRecentlyUsedPruner.java      |  221 --
 .../metron/dataloads/ThreatIntelBulkLoader.java |  213 --
 .../dataloads/bulk/LeastRecentlyUsedPruner.java |  221 ++
 .../dataloads/bulk/ThreatIntelBulkLoader.java   |  230 ++
 .../metron/dataloads/extractor/Extractor.java   |    4 +-
 .../dataloads/extractor/csv/CSVExtractor.java   |   20 +-
 .../extractor/csv/LookupConverter.java          |   30 +
 .../extractor/csv/LookupConverters.java         |   67 +
 .../dataloads/extractor/stix/StixExtractor.java |   44 +-
 .../extractor/stix/types/AddressHandler.java    |   18 +-
 .../extractor/stix/types/DomainHandler.java     |   26 +-
 .../extractor/stix/types/HostnameHandler.java   |   17 +-
 .../extractor/stix/types/ObjectTypeHandler.java |    3 +-
 .../dataloads/hbase/mr/BulkLoadMapper.java      |   21 +-
 .../metron/dataloads/hbase/mr/PrunerMapper.java |    5 +
 .../metron/dataloads/taxii/ConnectionType.java  |   23 +
 .../metron/dataloads/taxii/TableInfo.java       |   72 +
 .../dataloads/taxii/TaxiiConnectionConfig.java  |  196 ++
 .../metron/dataloads/taxii/TaxiiHandler.java    |  403 +++
 .../metron/dataloads/taxii/TaxiiLoader.java     |  180 ++
 .../dataloads/extractor/ExtractorTest.java      |   24 +-
 .../extractor/csv/CSVExtractorTest.java         |   17 +-
 .../extractor/stix/StixExtractorTest.java       |   25 +-
 .../dataloads/hbase/HBaseConverterTest.java     |   67 -
 .../hbase/HBaseThreatIntelConverterTest.java    |   76 +
 .../hbase/mr/BulkLoadMapperIntegrationTest.java |   24 +-
 .../dataloads/hbase/mr/BulkLoadMapperTest.java  |   19 +-
 .../LeastRecentlyUsedPrunerIntegrationTest.java |   32 +-
 .../dataloads/taxii/MockTaxiiService.java       |   99 +
 .../dataloads/taxii/TaxiiIntegrationTest.java   |  121 +
 .../resources/taxii-messages/message.discovery  |   21 +
 .../test/resources/taxii-messages/messages.poll | 2914 ++++++++++++++++++
 .../metron/threatintel/ThreatIntelAdapter.java  |    5 +-
 metron-streaming/Metron-Indexing/pom.xml        |    8 +-
 metron-streaming/Metron-Testing/pom.xml         |   88 +
 .../metron/integration/util/UnitTestHelper.java |   84 +
 .../util/integration/ComponentRunner.java       |  130 +
 .../util/integration/InMemoryComponent.java     |   23 +
 .../integration/util/integration/Processor.java |   23 +
 .../util/integration/ReadinessState.java        |   22 +
 .../integration/UnableToStartException.java     |   27 +
 .../components/ElasticSearchComponent.java      |  188 ++
 .../components/FluxTopologyComponent.java       |  132 +
 .../integration/util/mock/MockHTable.java       |  672 ++++
 metron-streaming/Metron-Topologies/pom.xml      |    6 +
 .../integration/pcap/PcapIntegrationTest.java   |   29 +-
 .../metron/integration/util/UnitTestHelper.java |   84 -
 .../util/integration/ComponentRunner.java       |  127 -
 .../util/integration/InMemoryComponent.java     |   23 -
 .../integration/util/integration/Processor.java |   23 -
 .../util/integration/ReadinessState.java        |   22 -
 .../integration/UnableToStartException.java     |   27 -
 .../components/ElasticSearchComponent.java      |  188 --
 .../components/FluxTopologyComponent.java       |  132 -
 .../integration/util/mock/MockHTable.java       |  673 ----
 .../util/threatintel/ThreatIntelHelper.java     |   12 +-
 metron-streaming/pom.xml                        |    1 +
 pom.xml                                         |    2 +
 76 files changed, 7034 insertions(+), 2330 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/NOTICE
----------------------------------------------------------------------
diff --git a/NOTICE b/NOTICE
index 9505518..ec99233 100644
--- a/NOTICE
+++ b/NOTICE
@@ -3,3 +3,6 @@
 
    This product includes software developed at
    The Apache Software Foundation (http://www.apache.org/).
+
+   This product includes Jersey Client: https://jersey.java.net/
+   License: Common Development and Distribution License (CDDL) v1.0 (https://glassfish.dev.java.net/public/CDDLv1.0.html)

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/pom.xml b/metron-streaming/Metron-Common/pom.xml
index c9918a2..57a58d7 100644
--- a/metron-streaming/Metron-Common/pom.xml
+++ b/metron-streaming/Metron-Common/pom.xml
@@ -132,6 +132,12 @@
         </dependency>
         <dependency>
             <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-common</artifactId>
+            <version>${global_hbase_version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
             <artifactId>hbase-client</artifactId>
             <version>${global_hbase_version}</version>
             <scope>provided</scope>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/AbstractConverter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/AbstractConverter.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/AbstractConverter.java
new file mode 100644
index 0000000..9072c22
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/AbstractConverter.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.hbase.converters;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.reference.lookup.LookupKV;
+import org.apache.metron.reference.lookup.LookupKey;
+import org.apache.metron.reference.lookup.LookupValue;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.*;
+
+
+public abstract class AbstractConverter<KEY_T extends LookupKey, VALUE_T extends LookupValue> implements HbaseConverter<KEY_T,VALUE_T> {
+    public static Function<Cell, Map.Entry<byte[], byte[]>> CELL_TO_ENTRY  = new Function<Cell, Map.Entry<byte[], byte[]>>() {
+
+        @Nullable
+        @Override
+        public Map.Entry<byte[], byte[]> apply(@Nullable Cell cell) {
+            return new AbstractMap.SimpleEntry<byte[], byte[]>(cell.getQualifier(), cell.getValue());
+        }
+    };
+    @Override
+    public Put toPut(String columnFamily, KEY_T key, VALUE_T values) throws IOException {
+        Put put = new Put(key.toBytes());
+        byte[] cf = Bytes.toBytes(columnFamily);
+        for(Map.Entry<byte[], byte[]> kv : values.toColumns()) {
+            put.add(cf, kv.getKey(), kv.getValue());
+        }
+        return put;
+    }
+
+    public LookupKV<KEY_T, VALUE_T> fromPut(Put put, String columnFamily, KEY_T key, VALUE_T value) throws IOException {
+        key.fromBytes(put.getRow());
+        byte[] cf = Bytes.toBytes(columnFamily);
+        value.fromColumns(Iterables.transform(put.getFamilyCellMap().get(cf), CELL_TO_ENTRY));
+        return new LookupKV<>(key, value);
+    }
+
+    @Override
+    public Result toResult(String columnFamily, KEY_T key, VALUE_T values) throws IOException {
+        Put put = toPut(columnFamily, key, values);
+        return Result.create(put.getFamilyCellMap().get(Bytes.toBytes(columnFamily)));
+    }
+
+    public LookupKV<KEY_T, VALUE_T> fromResult(Result result, String columnFamily, KEY_T key, VALUE_T value) throws IOException {
+        key.fromBytes(result.getRow());
+        byte[] cf = Bytes.toBytes(columnFamily);
+        NavigableMap<byte[], byte[]> cols = result.getFamilyMap(cf);
+        value.fromColumns(cols.entrySet());
+        return new LookupKV<>(key, value);
+    }
+    @Override
+    public Get toGet(String columnFamily, KEY_T key) {
+        Get ret = new Get(key.toBytes());
+        ret.addFamily(Bytes.toBytes(columnFamily));
+        return ret;
+    }
+
+    public static Iterable<Map.Entry<byte[], byte[]>> toEntries(byte[]... kvs) {
+        if(kvs.length % 2 != 0)  {
+            throw new IllegalStateException("Must be an even size");
+        }
+        List<Map.Entry<byte[], byte[]>> ret = new ArrayList<>(kvs.length/2);
+        for(int i = 0;i < kvs.length;i += 2) {
+            ret.add(new AbstractMap.SimpleImmutableEntry<>(kvs[i], kvs[i+1])) ;
+        }
+        return ret;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/HbaseConverter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/HbaseConverter.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/HbaseConverter.java
new file mode 100644
index 0000000..449d9cf
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/HbaseConverter.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.hbase.converters;
+
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.metron.reference.lookup.LookupKV;
+import org.apache.metron.reference.lookup.LookupKey;
+import org.apache.metron.reference.lookup.LookupValue;
+
+import java.io.IOException;
+
+public interface HbaseConverter<KEY_T extends LookupKey, VALUE_T extends LookupValue> {
+    Put toPut(String columnFamily, KEY_T key, VALUE_T values) throws IOException;
+
+    LookupKV<KEY_T, VALUE_T> fromPut(Put put, String columnFamily) throws IOException;
+
+    Result toResult(String columnFamily, KEY_T key, VALUE_T values) throws IOException;
+
+    LookupKV<KEY_T, VALUE_T> fromResult(Result result, String columnFamily) throws IOException;
+
+    Get toGet(String columnFamily, KEY_T key);
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/threatintel/ThreatIntelConverter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/threatintel/ThreatIntelConverter.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/threatintel/ThreatIntelConverter.java
new file mode 100644
index 0000000..d534a52
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/threatintel/ThreatIntelConverter.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.hbase.converters.threatintel;
+
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.metron.hbase.converters.AbstractConverter;
+import org.apache.metron.reference.lookup.LookupKV;
+
+import java.io.IOException;
+
+public class ThreatIntelConverter extends AbstractConverter<ThreatIntelKey, ThreatIntelValue> {
+
+    @Override
+    public LookupKV<ThreatIntelKey, ThreatIntelValue> fromPut(Put put, String columnFamily) throws IOException {
+        return fromPut(put, columnFamily, new ThreatIntelKey(), new ThreatIntelValue());
+    }
+
+    @Override
+    public LookupKV<ThreatIntelKey, ThreatIntelValue> fromResult(Result result, String columnFamily) throws IOException {
+        return fromResult(result, columnFamily, new ThreatIntelKey(), new ThreatIntelValue());
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/threatintel/ThreatIntelKey.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/threatintel/ThreatIntelKey.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/threatintel/ThreatIntelKey.java
new file mode 100644
index 0000000..3d898d9
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/threatintel/ThreatIntelKey.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.hbase.converters.threatintel;
+
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.reference.lookup.LookupKey;
+
+public class ThreatIntelKey implements LookupKey{
+    private static final int SEED = 0xDEADBEEF;
+    private static final int HASH_PREFIX_SIZE=16;
+    ThreadLocal<HashFunction> hFunction= new ThreadLocal<HashFunction>() {
+        @Override
+        protected HashFunction initialValue() {
+            return Hashing.murmur3_128(SEED);
+        }
+    };
+    public ThreatIntelKey() {
+
+    }
+    public ThreatIntelKey(String indicator) {
+        this.indicator = indicator;
+    }
+
+    public String indicator;
+
+    @Override
+    public byte[] toBytes() {
+        byte[] indicatorBytes = Bytes.toBytes(indicator);
+        Hasher hasher = hFunction.get().newHasher();
+        hasher.putBytes(Bytes.toBytes(indicator));
+        byte[] prefix = hasher.hash().asBytes();
+        byte[] val = new byte[indicatorBytes.length + prefix.length];
+        int pos = 0;
+        for(int i = 0;pos < prefix.length;++pos,++i) {
+            val[pos] = prefix[i];
+        }
+        for(int i = 0;i < indicatorBytes.length;++pos,++i) {
+            val[pos] = indicatorBytes[i];
+        }
+        return val;
+    }
+
+    @Override
+    public void fromBytes(byte[] row) {
+        ThreatIntelKey key = this;
+        key.indicator = Bytes.toString(row, HASH_PREFIX_SIZE, row.length - HASH_PREFIX_SIZE);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ThreatIntelKey that = (ThreatIntelKey) o;
+
+        return indicator != null ? indicator.equals(that.indicator) : that.indicator == null;
+
+    }
+
+    @Override
+    public int hashCode() {
+        return indicator != null ? indicator.hashCode() : 0;
+    }
+
+    @Override
+    public String toString() {
+        return "ThreatIntelKey{" +
+                "indicator='" + indicator + '\'' +
+                '}';
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/threatintel/ThreatIntelValue.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/threatintel/ThreatIntelValue.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/threatintel/ThreatIntelValue.java
new file mode 100644
index 0000000..97f0762
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/hbase/converters/threatintel/ThreatIntelValue.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.hbase.converters.threatintel;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.hbase.converters.AbstractConverter;
+import org.apache.metron.reference.lookup.LookupValue;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.type.TypeReference;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class ThreatIntelValue implements LookupValue {
+   private static final ThreadLocal<ObjectMapper> _mapper = new ThreadLocal<ObjectMapper>() {
+             @Override
+             protected ObjectMapper initialValue() {
+                return new ObjectMapper();
+             }
+    };
+    public static final String VALUE_COLUMN_NAME = "v";
+    public static final byte[] VALUE_COLUMN_NAME_B = Bytes.toBytes(VALUE_COLUMN_NAME);
+    public static final String LAST_SEEN_COLUMN_NAME = "t";
+    public static final byte[] LAST_SEEN_COLUMN_NAME_B = Bytes.toBytes(LAST_SEEN_COLUMN_NAME);
+
+    private Map<String, String> metadata = null;
+
+    public ThreatIntelValue()
+    {
+
+    }
+
+    public ThreatIntelValue(Map<String, String> metadata) {
+        this.metadata = metadata;
+    }
+
+
+
+    public Map<String, String> getMetadata() {
+        return metadata;
+    }
+
+    @Override
+    public Iterable<Map.Entry<byte[], byte[]>> toColumns() {
+        return AbstractConverter.toEntries( VALUE_COLUMN_NAME_B, Bytes.toBytes(valueToString(metadata))
+                                  );
+    }
+
+    @Override
+    public void fromColumns(Iterable<Map.Entry<byte[], byte[]>> values) {
+        for(Map.Entry<byte[], byte[]> cell : values) {
+            if(Bytes.equals(cell.getKey(), VALUE_COLUMN_NAME_B)) {
+                metadata = stringToValue(Bytes.toString(cell.getValue()));
+            }
+        }
+    }
+    public Map<String, String> stringToValue(String s){
+        try {
+            return _mapper.get().readValue(s, new TypeReference<Map<String, String>>(){});
+        } catch (IOException e) {
+            throw new RuntimeException("Unable to convert string to metadata: " + s);
+        }
+    }
+    public String valueToString(Map<String, String> value) {
+        try {
+            return _mapper.get().writeValueAsString(value);
+        } catch (IOException e) {
+            throw new RuntimeException("Unable to convert metadata to string: " + value);
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ThreatIntelValue that = (ThreatIntelValue) o;
+
+        return getMetadata() != null ? getMetadata().equals(that.getMetadata()) : that.getMetadata() == null;
+
+    }
+
+    @Override
+    public int hashCode() {
+        return getMetadata() != null ? getMetadata().hashCode() : 0;
+    }
+
+    @Override
+    public String toString() {
+        return "ThreatIntelValue{" +
+                "metadata=" + metadata +
+                '}';
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Common/src/main/java/org/apache/metron/reference/lookup/LookupKV.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/reference/lookup/LookupKV.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/reference/lookup/LookupKV.java
new file mode 100644
index 0000000..eb2b552
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/reference/lookup/LookupKV.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.reference.lookup;
+
+import java.io.Serializable;
+
+public class LookupKV<KEY_T extends LookupKey, VALUE_T extends LookupValue> implements Serializable {
+    private KEY_T key;
+    private VALUE_T value;
+    public LookupKV(KEY_T key, VALUE_T value) {
+        this.key = key;
+        this.value = value;
+    }
+
+    public KEY_T getKey() {
+        return key;
+    }
+
+    public VALUE_T getValue() {
+        return value;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        LookupKV<?, ?> lookupKV = (LookupKV<?, ?>) o;
+
+        if (key != null ? !key.equals(lookupKV.key) : lookupKV.key != null) return false;
+        return value != null ? value.equals(lookupKV.value) : lookupKV.value == null;
+
+    }
+
+    @Override
+    public int hashCode() {
+        int result = key != null ? key.hashCode() : 0;
+        result = 31 * result + (value != null ? value.hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "LookupKV{" +
+                "key=" + key +
+                ", value=" + value +
+                '}';
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Common/src/main/java/org/apache/metron/reference/lookup/LookupKey.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/reference/lookup/LookupKey.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/reference/lookup/LookupKey.java
index 8aecd64..c51ed9f 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/reference/lookup/LookupKey.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/reference/lookup/LookupKey.java
@@ -19,4 +19,5 @@ package org.apache.metron.reference.lookup;
 
 public interface LookupKey {
     byte[] toBytes();
+    void fromBytes(byte[] in);
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Common/src/main/java/org/apache/metron/reference/lookup/LookupValue.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/reference/lookup/LookupValue.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/reference/lookup/LookupValue.java
new file mode 100644
index 0000000..448f4c9
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/reference/lookup/LookupValue.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.metron.reference.lookup;
+
+import java.util.Map;
+import java.util.NavigableMap;
+
+public interface LookupValue {
+    Iterable<Map.Entry<byte[], byte[]>> toColumns();
+    void fromColumns(Iterable<Map.Entry<byte[], byte[]>> values);
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/ThreatIntelKey.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/ThreatIntelKey.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/ThreatIntelKey.java
deleted file mode 100644
index 8e90bc7..0000000
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/ThreatIntelKey.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.threatintel;
-
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hasher;
-import com.google.common.hash.Hashing;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.metron.reference.lookup.LookupKey;
-
-public class ThreatIntelKey implements LookupKey{
-    private static final int SEED = 0xDEADBEEF;
-    private static final int HASH_PREFIX_SIZE=16;
-    ThreadLocal<HashFunction> hFunction= new ThreadLocal<HashFunction>() {
-        @Override
-        protected HashFunction initialValue() {
-            return Hashing.murmur3_128(SEED);
-        }
-    };
-    public ThreatIntelKey() {
-
-    }
-    public ThreatIntelKey(String indicator) {
-        this.indicator = indicator;
-    }
-
-    public String indicator;
-
-    @Override
-    public byte[] toBytes() {
-        byte[] indicatorBytes = Bytes.toBytes(indicator);
-        Hasher hasher = hFunction.get().newHasher();
-        hasher.putBytes(Bytes.toBytes(indicator));
-        byte[] prefix = hasher.hash().asBytes();
-        byte[] val = new byte[indicatorBytes.length + prefix.length];
-        int pos = 0;
-        for(int i = 0;pos < prefix.length;++pos,++i) {
-            val[pos] = prefix[i];
-        }
-        for(int i = 0;i < indicatorBytes.length;++pos,++i) {
-            val[pos] = indicatorBytes[i];
-        }
-        return val;
-    }
-
-    public static ThreatIntelKey fromBytes(byte[] row) {
-        ThreatIntelKey key = new ThreatIntelKey();
-        key.indicator = Bytes.toString(row, HASH_PREFIX_SIZE, row.length - HASH_PREFIX_SIZE);
-        return key;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-
-        ThreatIntelKey that = (ThreatIntelKey) o;
-
-        return indicator != null ? indicator.equals(that.indicator) : that.indicator == null;
-
-    }
-
-    @Override
-    public int hashCode() {
-        return indicator != null ? indicator.hashCode() : 0;
-    }
-
-    @Override
-    public String toString() {
-        return "ThreatIntelKey{" +
-                "indicator='" + indicator + '\'' +
-                '}';
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/ThreatIntelResults.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/ThreatIntelResults.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/ThreatIntelResults.java
index 08f1cdc..8186b38 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/ThreatIntelResults.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/ThreatIntelResults.java
@@ -17,6 +17,8 @@
  */
 package org.apache.metron.threatintel;
 
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+
 import java.util.HashMap;
 import java.util.Map;
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/hbase/Converter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/hbase/Converter.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/hbase/Converter.java
deleted file mode 100644
index 89a846a..0000000
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/hbase/Converter.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.threatintel.hbase;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.metron.threatintel.ThreatIntelKey;
-import org.apache.metron.threatintel.ThreatIntelResults;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.type.TypeReference;
-
-import java.io.IOException;
-import java.util.*;
-
-public enum Converter {
-    INSTANCE;
-    public static final String VALUE_COLUMN_NAME = "v";
-    public static final byte[] VALUE_COLUMN_NAME_B = Bytes.toBytes(VALUE_COLUMN_NAME);
-    public static final String LAST_SEEN_COLUMN_NAME = "t";
-    public static final byte[] LAST_SEEN_COLUMN_NAME_B = Bytes.toBytes(LAST_SEEN_COLUMN_NAME);
-    private static final ThreadLocal<ObjectMapper> _mapper = new ThreadLocal<ObjectMapper>() {
-             @Override
-             protected ObjectMapper initialValue() {
-                return new ObjectMapper();
-             }
-    };
-    public Put toPut(String columnFamily, ThreatIntelKey key, Map<String, String> value, Long lastSeenTimestamp) throws IOException {
-        Put put = new Put(key.toBytes());
-        byte[] cf = Bytes.toBytes(columnFamily);
-        put.add(cf,VALUE_COLUMN_NAME_B, Bytes.toBytes(valueToString(value)));
-        put.add(cf, LAST_SEEN_COLUMN_NAME_B, Bytes.toBytes(lastSeenTimestamp));
-        return put;
-    }
-
-    public Map.Entry<ThreatIntelResults, Long> fromPut(Put put, String columnFamily) throws IOException {
-        ThreatIntelKey key = ThreatIntelKey.fromBytes(put.getRow());
-        Map<String, String> value = null;
-        Long lastSeen = null;
-        byte[] cf = Bytes.toBytes(columnFamily);
-        List<Cell> cells = put.getFamilyCellMap().get(cf);
-        for(Cell cell : cells) {
-            if(Bytes.equals(cell.getQualifier(), VALUE_COLUMN_NAME_B)) {
-                value = stringToValue(Bytes.toString(cell.getValue()));
-            }
-            else if(Bytes.equals(cell.getQualifier(), LAST_SEEN_COLUMN_NAME_B)) {
-               lastSeen = Bytes.toLong(cell.getValue());
-            }
-        }
-        return new AbstractMap.SimpleEntry<>(new ThreatIntelResults(key, value), lastSeen);
-    }
-
-    public Result toResult(String columnFamily, ThreatIntelKey key, Map<String, String> value, Long lastSeenTimestamp) throws IOException {
-        Put put = toPut(columnFamily, key, value, lastSeenTimestamp);
-        return Result.create(put.getFamilyCellMap().get(Bytes.toBytes(columnFamily)));
-    }
-
-    public Map.Entry<ThreatIntelResults, Long> fromResult(Result result, String columnFamily) throws IOException {
-        ThreatIntelKey key = ThreatIntelKey.fromBytes(result.getRow());
-        byte[] cf = Bytes.toBytes(columnFamily);
-        NavigableMap<byte[], byte[]> cols = result.getFamilyMap(cf);
-        Map<String, String> value = stringToValue(Bytes.toString(cols.get(VALUE_COLUMN_NAME_B)));
-        ThreatIntelResults results = new ThreatIntelResults(key, value);
-        return new AbstractMap.SimpleEntry<>(results, Bytes.toLong(cols.get(LAST_SEEN_COLUMN_NAME_B)));
-    }
-
-    public Get toGet(String columnFamily, ThreatIntelKey key) {
-        Get ret = new Get(key.toBytes());
-        ret.addFamily(Bytes.toBytes(columnFamily));
-        return ret;
-    }
-
-    public Map<String, String> stringToValue(String s) throws IOException {
-        return _mapper.get().readValue(s, new TypeReference<Map<String, String>>(){});
-    }
-    public String valueToString(Map<String, String> value) throws IOException {
-        return _mapper.get().writeValueAsString(value);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/hbase/ThreatIntelLookup.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/hbase/ThreatIntelLookup.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/hbase/ThreatIntelLookup.java
index c7997fe..13efc42 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/hbase/ThreatIntelLookup.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/threatintel/hbase/ThreatIntelLookup.java
@@ -17,36 +17,36 @@
  */
 package org.apache.metron.threatintel.hbase;
 
-import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.metron.hbase.converters.HbaseConverter;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelConverter;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
 import org.apache.metron.reference.lookup.Lookup;
-import org.apache.metron.reference.lookup.LookupKey;
+import org.apache.metron.reference.lookup.LookupKV;
 import org.apache.metron.reference.lookup.accesstracker.AccessTracker;
-import org.apache.metron.reference.lookup.handler.Handler;
-import org.apache.metron.threatintel.ThreatIntelKey;
-import org.apache.metron.threatintel.ThreatIntelResults;
 
-import java.io.Closeable;
 import java.io.IOException;
-import java.util.Map;
 
-public class ThreatIntelLookup extends Lookup<HTableInterface, ThreatIntelKey, Map.Entry<ThreatIntelResults, Long>> implements AutoCloseable {
 
+public class ThreatIntelLookup extends Lookup<HTableInterface, ThreatIntelKey, LookupKV<ThreatIntelKey,ThreatIntelValue>> implements AutoCloseable {
 
 
-    public static class Handler implements org.apache.metron.reference.lookup.handler.Handler<HTableInterface, ThreatIntelKey, Map.Entry<ThreatIntelResults, Long>> {
+
+    public static class Handler implements org.apache.metron.reference.lookup.handler.Handler<HTableInterface,ThreatIntelKey,LookupKV<ThreatIntelKey,ThreatIntelValue>> {
         String columnFamily;
+        HbaseConverter<ThreatIntelKey, ThreatIntelValue> converter = new ThreatIntelConverter();
         public Handler(String columnFamily) {
             this.columnFamily = columnFamily;
         }
         @Override
         public boolean exists(ThreatIntelKey key, HTableInterface table, boolean logAccess) throws IOException {
-            return table.exists(Converter.INSTANCE.toGet(columnFamily, key));
+            return table.exists(converter.toGet(columnFamily, key));
         }
 
         @Override
-        public Map.Entry<ThreatIntelResults,Long> get(ThreatIntelKey key, HTableInterface table, boolean logAccess) throws IOException {
-            return Converter.INSTANCE.fromResult(table.get(Converter.INSTANCE.toGet(columnFamily, key)), columnFamily);
+        public LookupKV<ThreatIntelKey, ThreatIntelValue> get(ThreatIntelKey key, HTableInterface table, boolean logAccess) throws IOException {
+            return converter.fromResult(table.get(converter.toGet(columnFamily, key)), columnFamily);
         }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/pom.xml b/metron-streaming/Metron-DataLoads/pom.xml
index d3eef64..5cac558 100644
--- a/metron-streaming/Metron-DataLoads/pom.xml
+++ b/metron-streaming/Metron-DataLoads/pom.xml
@@ -13,232 +13,270 @@
   -->
 
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>org.apache.metron</groupId>
-    <artifactId>Metron-Streaming</artifactId>
-    <version>0.1BETA</version>
-  </parent>
-  <artifactId>Metron-DataLoads</artifactId>
-  <properties>
-    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-  </properties>
-  <dependencies>
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-      <version>${global_hbase_guava_version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.mitre</groupId>
-      <artifactId>stix</artifactId>
-      <version>1.2.0.2</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.metron</groupId>
-      <artifactId>Metron-Common</artifactId>
-      <version>${project.parent.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>com.googlecode.disruptor</groupId>
-          <artifactId>disruptor</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <!--dependency>
-            <groupId>org.apache.storm</groupId>
-            <artifactId>storm-core</artifactId>
-            <version>${global_storm_version}</version>
-            <scope>provided</scope>
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.metron</groupId>
+        <artifactId>Metron-Streaming</artifactId>
+        <version>0.1BETA</version>
+    </parent>
+    <artifactId>Metron-DataLoads</artifactId>
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <httpcore.version>4.3.2</httpcore.version>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${global_hbase_guava_version}</version>
+        </dependency>
+        <dependency>
+            <groupId>javax.xml.bind</groupId>
+            <artifactId>jaxb-api</artifactId>
+            <version>2.2.11</version>
+        </dependency>
+        <dependency>
+            <groupId>net.sf.saxon</groupId>
+            <artifactId>Saxon-HE</artifactId>
+            <version>9.5.1-5</version>
             <exclusions>
                 <exclusion>
-                    <artifactId>servlet-api</artifactId>
-                    <groupId>javax.servlet</groupId>
+                    <groupId>javax.xml.bind</groupId>
+                    <artifactId>jaxb-api</artifactId>
                 </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.sun.xml.bind</groupId>
+            <artifactId>jaxb-impl</artifactId>
+            <version>2.2.5-2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.mitre</groupId>
+            <artifactId>stix</artifactId>
+            <version>1.2.0.2</version>
+            <exclusions>
                 <exclusion>
-                    <artifactId>log4j-over-slf4j</artifactId>
-                    <groupId>org.slf4j</groupId>
+                    <groupId>javax.xml.bind</groupId>
+                    <artifactId>jaxb-api</artifactId>
                 </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>Metron-Common</artifactId>
+            <version>${project.parent.version}</version>
+            <exclusions>
                 <exclusion>
                     <groupId>com.googlecode.disruptor</groupId>
                     <artifactId>disruptor</artifactId>
                 </exclusion>
             </exclusions>
-        </dependency-->
+        </dependency>
         <dependency>
-          <groupId>org.apache.hbase</groupId>
-          <artifactId>hbase-client</artifactId>
-          <version>${global_hbase_version}</version>
-          <scope>provided</scope>
-          <exclusions>
-            <exclusion>
-              <artifactId>log4j</artifactId>
-              <groupId>log4j</groupId>
-            </exclusion>
-          </exclusions>
+            <groupId>org.mitre.taxii</groupId>
+            <artifactId>taxii</artifactId>
+            <version>1.1.0.1</version>
+            <!--scope>provided</scope-->
         </dependency>
         <dependency>
-          <groupId>org.apache.hbase</groupId>
-          <artifactId>hbase-server</artifactId>
-          <version>${global_hbase_version}</version>
-          <scope>provided</scope>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-common</artifactId>
+            <version>${global_hbase_version}</version>
+            <scope>provided</scope>
         </dependency>
         <dependency>
-          <groupId>com.opencsv</groupId>
-          <artifactId>opencsv</artifactId>
-          <version>${global_opencsv_version}</version>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-client</artifactId>
+            <version>${global_hbase_version}</version>
+            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <artifactId>log4j</artifactId>
+                    <groupId>log4j</groupId>
+                </exclusion>
+            </exclusions>
         </dependency>
-
         <dependency>
-          <groupId>org.apache.hbase</groupId>
-          <artifactId>hbase-testing-util</artifactId>
-          <version>${global_hbase_version}</version>
-          <scope>test</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>org.slf4j</groupId>
-              <artifactId>slf4j-log4j12</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-        <!--dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>${global_guava_version}</version>
-        </dependency-->
-
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-server</artifactId>
+            <version>${global_hbase_version}</version>
+            <scope>provided</scope>
+        </dependency>
         <dependency>
-          <groupId>org.slf4j</groupId>
-          <artifactId>log4j-over-slf4j</artifactId>
-          <version>${global_slf4j_version}</version>
-          <scope>test</scope>
+            <groupId>com.opencsv</groupId>
+            <artifactId>opencsv</artifactId>
+            <version>${global_opencsv_version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpcore</artifactId>
+            <version>${httpcore.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>${httpcore.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-testing-util</artifactId>
+            <version>${global_hbase_version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.sun.jersey</groupId>
+            <artifactId>jersey-client</artifactId>
+            <version>1.19</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>log4j-over-slf4j</artifactId>
+            <version>${global_slf4j_version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>Metron-Testing</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
         </dependency>
 
-
-      </dependencies>
-      <build>
+    </dependencies>
+    <build>
         <resources>
-          <resource>
-            <directory>src</directory>
-            <excludes>
-              <exclude>**/*.java</exclude>
-            </excludes>
-          </resource>
+            <resource>
+                <directory>src</directory>
+                <excludes>
+                    <exclude>**/*.java</exclude>
+                </excludes>
+            </resource>
         </resources>
         <plugins>
-          <plugin>
-            <!-- Separates the unit tests from the integration tests. -->
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-surefire-plugin</artifactId>
-            <version>2.12.4</version>
-            <configuration>
-              <!-- Skip the default running of this plug-in (or everything is run twice...see below) -->
-              <argLine>-Xmx2048m -XX:MaxPermSize=256m</argLine>
-              <skip>true</skip>
-              <!-- Show 100% of the lines from the stack trace (doesn't work) -->
-              <trimStackTrace>false</trimStackTrace>
-
-            </configuration>
-            <executions>
-              <execution>
-                <id>unit-tests</id>
-                <phase>test</phase>
-                <goals>
-                  <goal>test</goal>
-                </goals>
+            <plugin>
+                <!-- Separates the unit tests from the integration tests. -->
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>2.12.4</version>
                 <configuration>
-                  <!-- Never skip running the tests when the test phase is invoked -->
-                  <skip>false</skip>
-                  <includes>
-                    <!-- Include unit tests within integration-test phase. -->
-                    <include>**/*Test.java</include>
-                  </includes>
-                  <excludes>
-                    <!-- Exclude integration tests within (unit) test phase. -->
-                    <exclude>**/*IntegrationTest.java</exclude>
-                  </excludes>
+                    <!-- Skip the default running of this plug-in (or everything is run twice...see below) -->
+                    <argLine>-Xmx2048m -XX:MaxPermSize=256m</argLine>
+                    <skip>true</skip>
+                    <!-- Show 100% of the lines from the stack trace (doesn't work) -->
+                    <trimStackTrace>false</trimStackTrace>
+
                 </configuration>
-              </execution>
-              <execution>
-                <id>integration-tests</id>
-                <phase>integration-test</phase>
-                <goals>
-                  <goal>test</goal>
-                </goals>
+                <executions>
+                    <execution>
+                        <id>unit-tests</id>
+                        <phase>test</phase>
+                        <goals>
+                            <goal>test</goal>
+                        </goals>
+                        <configuration>
+                            <!-- Never skip running the tests when the test phase is invoked -->
+                            <skip>false</skip>
+                            <includes>
+                                <!-- Include unit tests within integration-test phase. -->
+                                <include>**/*Test.java</include>
+                            </includes>
+                            <excludes>
+                                <!-- Exclude integration tests within (unit) test phase. -->
+                                <exclude>**/*IntegrationTest.java</exclude>
+                            </excludes>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>integration-tests</id>
+                        <phase>integration-test</phase>
+                        <goals>
+                            <goal>test</goal>
+                        </goals>
+                        <configuration>
+                            <!-- Never skip running the tests when the integration-test phase is invoked -->
+                            <skip>false</skip>
+                            <includes>
+                                <!-- Include integration tests within integration-test phase. -->
+                                <include>**/*IntegrationTest.java</include>
+                            </includes>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.1</version>
                 <configuration>
-                  <!-- Never skip running the tests when the integration-test phase is invoked -->
-                  <skip>false</skip>
-                  <includes>
-                    <!-- Include integration tests within integration-test phase. -->
-                    <include>**/*IntegrationTest.java</include>
-                  </includes>
+                    <source>1.7</source>
+                    <target>1.7</target>
                 </configuration>
-              </execution>
-            </executions>
-          </plugin>
-
-          <plugin>
-            <artifactId>maven-compiler-plugin</artifactId>
-            <version>3.1</version>
-            <configuration>
-              <source>1.7</source>
-              <target>1.7</target>
-            </configuration>
-          </plugin>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-shade-plugin</artifactId>
-            <version>2.3</version>
-            <executions>
-              <execution>
-                <phase>package</phase>
-                <goals>
-                  <goal>shade</goal>
-                </goals>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>2.3</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <relocations>
+                                <relocation>
+                                    <pattern>com.google.common</pattern>
+                                    <shadedPattern>org.apache.metron.guava.dataload</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>org.apache.http</pattern>
+                                    <shadedPattern>org.apache.metron.httpcore.dataload</shadedPattern>
+                                </relocation>
+                            </relocations>
+                            <!--minimizeJar>true</minimizeJar-->
+                            <artifactSet>
+                                <excludes>
+                                    <exclude>classworlds:classworlds</exclude>
+                                    <exclude>junit:junit</exclude>
+                                    <exclude>jmock:*</exclude>
+                                    <exclude>*:xml-apis</exclude>
+                                    <exclude>*slf4j*</exclude>
+                                    <exclude>org.apache.maven:lib:tests</exclude>
+                                    <exclude>log4j:log4j:jar:</exclude>
+                                    <exclude>*:hbase:*</exclude>
+                                    <exclude>org.apache.hadoop.yarn.util.package-info*</exclude>
+                                </excludes>
+                            </artifactSet>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
                 <configuration>
-                  <relocations>
-                    <relocation>
-                      <pattern>com.google.common</pattern>
-                      <shadedPattern>org.apache.metron.guava.dataload</shadedPattern>
-                    </relocation>
-                  </relocations>
-                  <minimizeJar>true</minimizeJar>
-                  <artifactSet>
-                    <excludes>
-                      <exclude>classworlds:classworlds</exclude>
-                      <exclude>junit:junit</exclude>
-                      <exclude>jmock:*</exclude>
-                      <exclude>*:xml-apis</exclude>
-                      <exclude>*slf4j*</exclude>
-                      <exclude>org.apache.maven:lib:tests</exclude>
-                      <exclude>log4j:log4j:jar:</exclude>
-                      <exclude>*:hbase:*</exclude>
-                      <exclude>org.apache.hadoop.yarn.util.package-info*</exclude>
-                    </excludes>
-                  </artifactSet>
+                    <descriptor>src/main/assembly/assembly.xml</descriptor>
                 </configuration>
-              </execution>
-            </executions>
-          </plugin>
-          <plugin>
-            <artifactId>maven-assembly-plugin</artifactId>
-            <configuration>
-              <descriptor>src/main/assembly/assembly.xml</descriptor>
-            </configuration>
-            <executions>
-              <execution>
-                <id>make-assembly</id> <!-- this is used for inheritance merges -->
-                <phase>package</phase> <!-- bind to the packaging phase -->
-                <goals>
-                  <goal>single</goal>
-                </goals>
-              </execution>
-            </executions>
-          </plugin>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id> <!-- this is used for inheritance merges -->
+                        <phase>package</phase> <!-- bind to the packaging phase -->
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
 
         </plugins>
-      </build>
+    </build>
 
-    </project>
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/bash/threatintel_bulk_load.sh
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/bash/threatintel_bulk_load.sh b/metron-streaming/Metron-DataLoads/src/main/bash/threatintel_bulk_load.sh
index 0b7a83c..32b07f3 100755
--- a/metron-streaming/Metron-DataLoads/src/main/bash/threatintel_bulk_load.sh
+++ b/metron-streaming/Metron-DataLoads/src/main/bash/threatintel_bulk_load.sh
@@ -35,4 +35,4 @@ for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
   fi
 done
 export HADOOP_CLASSPATH
-hadoop jar /usr/metron/0.1BETA/lib/Metron-DataLoads-0.1BETA.jar org.apache.metron.dataloads.ThreatIntelBulkLoader -libjars ${LIBJARS} "$@"
+hadoop jar /usr/metron/0.1BETA/lib/Metron-DataLoads-0.1BETA.jar org.apache.metron.dataloads.bulk.ThreatIntelBulkLoader -libjars ${LIBJARS} "$@"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/bash/threatintel_bulk_prune.sh
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/bash/threatintel_bulk_prune.sh b/metron-streaming/Metron-DataLoads/src/main/bash/threatintel_bulk_prune.sh
index ee230c2..88cd946 100755
--- a/metron-streaming/Metron-DataLoads/src/main/bash/threatintel_bulk_prune.sh
+++ b/metron-streaming/Metron-DataLoads/src/main/bash/threatintel_bulk_prune.sh
@@ -34,4 +34,4 @@ for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
   fi
 done
 export HADOOP_CLASSPATH
-hadoop jar /usr/metron/0.1BETA/lib/Metron-DataLoads-0.1BETA.jar org.apache.metron.dataloads.LeastRecentlyUsedPruner -libjars ${LIBJARS} "$@"
+hadoop jar /usr/metron/0.1BETA/lib/Metron-DataLoads-0.1BETA.jar org.apache.metron.dataloads.bulk.LeastRecentlyUsedPruner -libjars ${LIBJARS} "$@"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/bash/threatintel_taxii_load.sh
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/bash/threatintel_taxii_load.sh b/metron-streaming/Metron-DataLoads/src/main/bash/threatintel_taxii_load.sh
new file mode 100755
index 0000000..77011fb
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/main/bash/threatintel_taxii_load.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# 
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+
+BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
+[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
+
+# Autodetect JAVA_HOME if not defined
+if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+  . /usr/libexec/bigtop-detect-javahome
+elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+  . /usr/lib/bigtop-utils/bigtop-detect-javahome
+fi
+
+export HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client}
+CP=/usr/metron/0.1BETA/lib/Metron-DataLoads-0.1BETA.jar:/usr/metron/0.1BETA/lib/taxii-1.1.0.1.jar:`${HBASE_HOME}/bin/hbase classpath`
+HADOOP_CLASSPATH=$(echo $CP )
+for jar in $(echo $HADOOP_CLASSPATH | sed 's/:/ /g');do
+  if [ -f $jar ];then
+    LIBJARS="$jar,$LIBJARS"
+  fi
+done
+export HADOOP_CLASSPATH
+hadoop jar /usr/metron/0.1BETA/lib/Metron-DataLoads-0.1BETA.jar org.apache.metron.dataloads.taxii.TaxiiLoader "$@"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/LeastRecentlyUsedPruner.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/LeastRecentlyUsedPruner.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/LeastRecentlyUsedPruner.java
deleted file mode 100644
index 4ee6e2f..0000000
--- a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/LeastRecentlyUsedPruner.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.dataloads;
-
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import org.apache.commons.cli.*;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.metron.dataloads.hbase.mr.PrunerMapper;
-
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-public class LeastRecentlyUsedPruner {
-    private static abstract class OptionHandler implements Function<String, Option> {}
-    private enum BulkLoadOptions {
-        HELP("h", new OptionHandler() {
-
-            @Nullable
-            @Override
-            public Option apply(@Nullable String s) {
-                return new Option(s, "help", false, "Generate Help screen");
-            }
-        }), TABLE("t", new OptionHandler() {
-            @Nullable
-            @Override
-            public Option apply(@Nullable String s) {
-                Option o = new Option(s, "table", true, "HBase table to prune");
-                o.setRequired(true);
-                o.setArgName("HBASE_TABLE");
-                return o;
-            }
-        }), COLUMN_FAMILY("f", new OptionHandler() {
-            @Nullable
-            @Override
-            public Option apply(@Nullable String s) {
-                Option o = new Option(s, "column_family", true, "Column family of the HBase table to prune");
-                o.setRequired(false);
-                o.setArgName("CF_NAME");
-                return o;
-            }
-        })
-        ,AS_OF_TIME("a", new OptionHandler() {
-            @Nullable
-            @Override
-            public Option apply(@Nullable String s) {
-                Option o = new Option(s, "as_of", true, "The earliest access tracker you want to use.");
-                o.setArgName("datetime");
-                o.setRequired(true);
-                return o;
-            }
-        })
-        ,AS_OF_TIME_FORMAT("t", new OptionHandler() {
-            @Nullable
-            @Override
-            public Option apply(@Nullable String s) {
-                String defaultFormat = new SimpleDateFormat().toLocalizedPattern();
-                Option o = new Option(s, "as_of_format", true, "The format of the as_of time (only used in conjunction with the as_of option) (Default is: " + defaultFormat + ")");
-                o.setArgName("format");
-                o.setRequired(false);
-                return o;
-            }
-        })
-        ,ACCESS_TABLE("u", new OptionHandler() {
-            @Nullable
-            @Override
-            public Option apply(@Nullable String s) {
-                Option o = new Option(s, "access_table", true, "HBase table containing the access trackers.");
-                o.setRequired(true);
-                o.setArgName("HBASE_TABLE");
-                return o;
-            }
-        }), ACCESS_COLUMN_FAMILY("z", new OptionHandler() {
-            @Nullable
-            @Override
-            public Option apply(@Nullable String s) {
-                Option o = new Option(s, "access_column_family", true, "Column family of the HBase table containing the access trackers");
-                o.setRequired(true);
-                o.setArgName("CF_NAME");
-                return o;
-            }
-        });
-        Option option;
-        String shortCode;
-        BulkLoadOptions(String shortCode, OptionHandler optionHandler) {
-            this.shortCode = shortCode;
-            this.option = optionHandler.apply(shortCode);
-        }
-
-        public boolean has(CommandLine cli) {
-            return cli.hasOption(shortCode);
-        }
-
-        public String get(CommandLine cli) {
-            return cli.getOptionValue(shortCode);
-        }
-        private static long getTimestamp(CommandLine cli) throws java.text.ParseException {
-            Date d = getFormat(cli).parse(BulkLoadOptions.AS_OF_TIME.get(cli));
-            return d.getTime();
-        }
-
-        private static DateFormat getFormat(CommandLine cli) {
-            DateFormat format = new SimpleDateFormat();
-            if (BulkLoadOptions.AS_OF_TIME_FORMAT.has(cli)) {
-                 format = new SimpleDateFormat(BulkLoadOptions.AS_OF_TIME_FORMAT.get(cli));
-            }
-            return format;
-        }
-
-        public static CommandLine parse(CommandLineParser parser, String[] args) {
-            try {
-                CommandLine cli = parser.parse(getOptions(), args);
-                if(BulkLoadOptions.HELP.has(cli)) {
-                    printHelp();
-                    System.exit(0);
-                }
-                return cli;
-            } catch (ParseException e) {
-                System.err.println("Unable to parse args: " + Joiner.on(' ').join(args));
-                e.printStackTrace(System.err);
-                printHelp();
-                System.exit(-1);
-                return null;
-            }
-        }
-
-        public static void printHelp() {
-            HelpFormatter formatter = new HelpFormatter();
-            formatter.printHelp( "ThreatIntelBulkLoader", getOptions());
-        }
-
-        public static Options getOptions() {
-            Options ret = new Options();
-            for(BulkLoadOptions o : BulkLoadOptions.values()) {
-               ret.addOption(o.option);
-            }
-            return ret;
-        }
-    }
-
-    public static void setupHBaseJob(Job job, String sourceTable, String cf) throws IOException {
-        Scan scan = new Scan();
-        if(cf != null) {
-            scan.addFamily(Bytes.toBytes(cf));
-        }
-        scan.setCaching(500);        // 1 is the default in Scan, which will be bad for MapReduce jobs
-        scan.setCacheBlocks(false);  // don't set to true for MR jobs
-// set other scan attrs
-
-        TableMapReduceUtil.initTableMapperJob(
-                sourceTable,      // input table
-                scan,	          // Scan instance to control CF and attribute selection
-                PrunerMapper.class,   // mapper class
-                null,	          // mapper output key
-                null,	          // mapper output value
-                job);
-        TableMapReduceUtil.initTableReducerJob(
-                sourceTable,      // output table
-                null,             // reducer class
-                job);
-    }
-
-    public static Job createJob( Configuration conf
-                               , String table
-                               , String cf
-                               , String accessTrackerTable
-                               , String accessTrackerColumnFamily
-                               , Long ts
-                               ) throws IOException
-    {
-        Job job = new Job(conf);
-        job.setJobName("LeastRecentlyUsedPruner: Pruning " +  table + ":" + cf + " since " + new SimpleDateFormat().format(new Date(ts)));
-        System.out.println("Configuring " + job.getJobName());
-        job.setJarByClass(LeastRecentlyUsedPruner.class);
-        job.getConfiguration().setLong(PrunerMapper.TIMESTAMP_CONF, ts);
-        job.getConfiguration().set(PrunerMapper.ACCESS_TRACKER_NAME_CONF, table);
-        job.getConfiguration().set(PrunerMapper.ACCESS_TRACKER_CF_CONF, accessTrackerColumnFamily);
-        job.getConfiguration().set(PrunerMapper.ACCESS_TRACKER_TABLE_CONF, accessTrackerTable);
-        setupHBaseJob(job, table, cf);
-        job.setNumReduceTasks(0);
-        return job;
-    }
-
-    public static void main(String... argv) throws IOException, java.text.ParseException, ClassNotFoundException, InterruptedException {
-        Configuration conf = HBaseConfiguration.create();
-        String[] otherArgs = new GenericOptionsParser(conf, argv).getRemainingArgs();
-
-        CommandLine cli = BulkLoadOptions.parse(new PosixParser(), otherArgs);
-        Long ts = BulkLoadOptions.getTimestamp(cli);
-        String table = BulkLoadOptions.TABLE.get(cli);
-        String cf = BulkLoadOptions.COLUMN_FAMILY.get(cli);
-        String accessTrackerTable = BulkLoadOptions.ACCESS_TABLE.get(cli);
-        String accessTrackerCF = BulkLoadOptions.ACCESS_COLUMN_FAMILY.get(cli);
-        Job job = createJob(conf, table, cf, accessTrackerTable, accessTrackerCF, ts);
-        System.exit(job.waitForCompletion(true) ? 0 : 1);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/ThreatIntelBulkLoader.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/ThreatIntelBulkLoader.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/ThreatIntelBulkLoader.java
deleted file mode 100644
index 246924c..0000000
--- a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/ThreatIntelBulkLoader.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.dataloads;
-
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.io.Files;
-import org.apache.commons.cli.*;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.metron.dataloads.extractor.ExtractorHandler;
-import org.apache.metron.dataloads.hbase.mr.BulkLoadMapper;
-
-import javax.annotation.Nullable;
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.text.*;
-import java.util.Date;
-
-public class ThreatIntelBulkLoader  {
-    private static abstract class OptionHandler implements Function<String, Option> {}
-    private enum BulkLoadOptions {
-        HELP("h", new OptionHandler() {
-
-            @Nullable
-            @Override
-            public Option apply(@Nullable String s) {
-                return new Option(s, "help", false, "Generate Help screen");
-            }
-        })
-        ,TABLE("t", new OptionHandler() {
-            @Nullable
-            @Override
-            public Option apply(@Nullable String s) {
-                Option o = new Option(s, "table", true, "HBase table to import data into");
-                o.setRequired(true);
-                o.setArgName("HBASE_TABLE");
-                return o;
-            }
-        })
-        ,COLUMN_FAMILY("f", new OptionHandler() {
-            @Nullable
-            @Override
-            public Option apply(@Nullable String s) {
-                Option o = new Option(s, "column_family", true, "Column family of the HBase table to import into");
-                o.setRequired(true);
-                o.setArgName("CF_NAME");
-                return o;
-            }
-        })
-        ,EXTRACTOR_CONFIG("e", new OptionHandler() {
-            @Nullable
-            @Override
-            public Option apply(@Nullable String s) {
-                Option o = new Option(s, "extractor_config", true, "JSON Document describing the extractor for this input data source");
-                o.setArgName("JSON_FILE");
-                o.setRequired(true);
-                return o;
-            }
-        })
-        ,INPUT_DATA("i", new OptionHandler() {
-            @Nullable
-            @Override
-            public Option apply(@Nullable String s) {
-                Option o = new Option(s, "input", true, "Input directory in HDFS for the data to import into HBase");
-                o.setArgName("DIR");
-                o.setRequired(true);
-                return o;
-            }
-        })
-        ,AS_OF_TIME("a", new OptionHandler() {
-            @Nullable
-            @Override
-            public Option apply(@Nullable String s) {
-                Option o = new Option(s, "as_of", true, "The last read timestamp to mark the records with (omit for time of execution)");
-                o.setArgName("datetime");
-                o.setRequired(false);
-                return o;
-            }
-        })
-        ,AS_OF_TIME_FORMAT("t", new OptionHandler() {
-            @Nullable
-            @Override
-            public Option apply(@Nullable String s) {
-                Option o = new Option(s, "as_of_format", true, "The format of the as_of time (only used in conjunction with the as_of option)");
-                o.setArgName("format");
-                o.setRequired(false);
-                return o;
-            }
-        })
-        ;
-        Option option;
-        String shortCode;
-        BulkLoadOptions(String shortCode, OptionHandler optionHandler) {
-            this.shortCode = shortCode;
-            this.option = optionHandler.apply(shortCode);
-        }
-
-        public boolean has(CommandLine cli) {
-            return cli.hasOption(shortCode);
-        }
-
-        public String get(CommandLine cli) {
-            return cli.getOptionValue(shortCode);
-        }
-
-        public static CommandLine parse(CommandLineParser parser, String[] args) {
-            try {
-                CommandLine cli = parser.parse(getOptions(), args);
-                if(ThreatIntelBulkLoader.BulkLoadOptions.HELP.has(cli)) {
-                    printHelp();
-                    System.exit(0);
-                }
-                return cli;
-            } catch (ParseException e) {
-                System.err.println("Unable to parse args: " + Joiner.on(' ').join(args));
-                e.printStackTrace(System.err);
-                printHelp();
-                System.exit(-1);
-                return null;
-            }
-        }
-
-        public static void printHelp() {
-            HelpFormatter formatter = new HelpFormatter();
-            formatter.printHelp( "ThreatIntelBulkLoader", getOptions());
-        }
-
-        public static Options getOptions() {
-            Options ret = new Options();
-            for(BulkLoadOptions o : BulkLoadOptions.values()) {
-               ret.addOption(o.option);
-            }
-            return ret;
-        }
-    }
-    private static long getTimestamp(CommandLine cli) throws java.text.ParseException {
-        if(BulkLoadOptions.AS_OF_TIME.has(cli)) {
-            if(!BulkLoadOptions.AS_OF_TIME_FORMAT.has(cli)) {
-                throw new IllegalStateException("Unable to proceed: Specified as_of_time without an associated format.");
-            }
-            else {
-                DateFormat format = new SimpleDateFormat(BulkLoadOptions.AS_OF_TIME_FORMAT.get(cli));
-                Date d = format.parse(BulkLoadOptions.AS_OF_TIME.get(cli));
-                return d.getTime();
-            }
-        }
-        else {
-            return System.currentTimeMillis();
-        }
-    }
-    private static String readExtractorConfig(File configFile) throws IOException {
-        return Joiner.on("\n").join(Files.readLines(configFile, Charset.defaultCharset()));
-    }
-
-    public static Job createJob(Configuration conf, String input, String table, String cf, String extractorConfigContents, long ts) throws IOException {
-        Job job = new Job(conf);
-        job.setJobName("ThreatIntelBulkLoader: " + input + " => " +  table + ":" + cf);
-        System.out.println("Configuring " + job.getJobName());
-        job.setJarByClass(ThreatIntelBulkLoader.class);
-        job.setMapperClass(org.apache.metron.dataloads.hbase.mr.BulkLoadMapper.class);
-        job.setOutputFormatClass(TableOutputFormat.class);
-        job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, table);
-        job.getConfiguration().set(BulkLoadMapper.COLUMN_FAMILY_KEY, cf);
-        job.getConfiguration().set(BulkLoadMapper.CONFIG_KEY, extractorConfigContents);
-        job.getConfiguration().set(BulkLoadMapper.LAST_SEEN_KEY, "" + ts);
-        job.setOutputKeyClass(ImmutableBytesWritable.class);
-        job.setOutputValueClass(Put.class);
-        job.setNumReduceTasks(0);
-        ExtractorHandler handler = ExtractorHandler.load(extractorConfigContents);
-        handler.getInputFormatHandler().set(job, new Path(input), handler.getConfig());
-        return job;
-    }
-
-    public static void main(String... argv) throws IOException, java.text.ParseException, ClassNotFoundException, InterruptedException {
-        Configuration conf = HBaseConfiguration.create();
-        String[] otherArgs = new GenericOptionsParser(conf, argv).getRemainingArgs();
-
-        CommandLine cli = BulkLoadOptions.parse(new PosixParser(), otherArgs);
-        Long ts = getTimestamp(cli);
-        String input = BulkLoadOptions.INPUT_DATA.get(cli);
-        String table = BulkLoadOptions.TABLE.get(cli);
-        String cf = BulkLoadOptions.COLUMN_FAMILY.get(cli);
-        String extractorConfigContents = readExtractorConfig(new File(BulkLoadOptions.EXTRACTOR_CONFIG.get(cli)));
-        Job job = createJob(conf, input, table, cf, extractorConfigContents, ts);
-        System.out.println(conf);
-        System.exit(job.waitForCompletion(true) ? 0 : 1);
-    }
-}


[5/6] incubator-metron git commit: METRON-50: Ingest threat intel data from Taxii feeds closes apache/incubator-metron#29

Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/bulk/LeastRecentlyUsedPruner.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/bulk/LeastRecentlyUsedPruner.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/bulk/LeastRecentlyUsedPruner.java
new file mode 100644
index 0000000..8f1f205
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/bulk/LeastRecentlyUsedPruner.java
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.dataloads.bulk;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import org.apache.commons.cli.*;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.metron.dataloads.hbase.mr.PrunerMapper;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+public class LeastRecentlyUsedPruner {
+    private static abstract class OptionHandler implements Function<String, Option> {}
+    private enum BulkLoadOptions {
+        HELP("h", new OptionHandler() {
+
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                return new Option(s, "help", false, "Generate Help screen");
+            }
+        }), TABLE("t", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "table", true, "HBase table to prune");
+                o.setRequired(true);
+                o.setArgName("HBASE_TABLE");
+                return o;
+            }
+        }), COLUMN_FAMILY("f", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "column_family", true, "Column family of the HBase table to prune");
+                o.setRequired(false);
+                o.setArgName("CF_NAME");
+                return o;
+            }
+        })
+        ,AS_OF_TIME("a", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "as_of", true, "The earliest access tracker you want to use.");
+                o.setArgName("datetime");
+                o.setRequired(true);
+                return o;
+            }
+        })
+        ,AS_OF_TIME_FORMAT("t", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                String defaultFormat = new SimpleDateFormat().toLocalizedPattern();
+                Option o = new Option(s, "as_of_format", true, "The format of the as_of time (only used in conjunction with the as_of option) (Default is: " + defaultFormat + ")");
+                o.setArgName("format");
+                o.setRequired(false);
+                return o;
+            }
+        })
+        ,ACCESS_TABLE("u", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "access_table", true, "HBase table containing the access trackers.");
+                o.setRequired(true);
+                o.setArgName("HBASE_TABLE");
+                return o;
+            }
+        }), ACCESS_COLUMN_FAMILY("z", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "access_column_family", true, "Column family of the HBase table containing the access trackers");
+                o.setRequired(true);
+                o.setArgName("CF_NAME");
+                return o;
+            }
+        });
+        Option option;
+        String shortCode;
+        BulkLoadOptions(String shortCode, OptionHandler optionHandler) {
+            this.shortCode = shortCode;
+            this.option = optionHandler.apply(shortCode);
+        }
+
+        public boolean has(CommandLine cli) {
+            return cli.hasOption(shortCode);
+        }
+
+        public String get(CommandLine cli) {
+            return cli.getOptionValue(shortCode);
+        }
+        private static long getTimestamp(CommandLine cli) throws java.text.ParseException {
+            Date d = getFormat(cli).parse(BulkLoadOptions.AS_OF_TIME.get(cli));
+            return d.getTime();
+        }
+
+        private static DateFormat getFormat(CommandLine cli) {
+            DateFormat format = new SimpleDateFormat();
+            if (BulkLoadOptions.AS_OF_TIME_FORMAT.has(cli)) {
+                 format = new SimpleDateFormat(BulkLoadOptions.AS_OF_TIME_FORMAT.get(cli));
+            }
+            return format;
+        }
+
+        public static CommandLine parse(CommandLineParser parser, String[] args) {
+            try {
+                CommandLine cli = parser.parse(getOptions(), args);
+                if(BulkLoadOptions.HELP.has(cli)) {
+                    printHelp();
+                    System.exit(0);
+                }
+                return cli;
+            } catch (ParseException e) {
+                System.err.println("Unable to parse args: " + Joiner.on(' ').join(args));
+                e.printStackTrace(System.err);
+                printHelp();
+                System.exit(-1);
+                return null;
+            }
+        }
+
+        public static void printHelp() {
+            HelpFormatter formatter = new HelpFormatter();
+            formatter.printHelp( "ThreatIntelBulkLoader", getOptions());
+        }
+
+        public static Options getOptions() {
+            Options ret = new Options();
+            for(BulkLoadOptions o : BulkLoadOptions.values()) {
+               ret.addOption(o.option);
+            }
+            return ret;
+        }
+    }
+
+    public static void setupHBaseJob(Job job, String sourceTable, String cf) throws IOException {
+        Scan scan = new Scan();
+        if(cf != null) {
+            scan.addFamily(Bytes.toBytes(cf));
+        }
+        scan.setCaching(500);        // 1 is the default in Scan, which will be bad for MapReduce jobs
+        scan.setCacheBlocks(false);  // don't set to true for MR jobs
+// set other scan attrs
+
+        TableMapReduceUtil.initTableMapperJob(
+                sourceTable,      // input table
+                scan,	          // Scan instance to control CF and attribute selection
+                PrunerMapper.class,   // mapper class
+                null,	          // mapper output key
+                null,	          // mapper output value
+                job);
+        TableMapReduceUtil.initTableReducerJob(
+                sourceTable,      // output table
+                null,             // reducer class
+                job);
+    }
+
+    public static Job createJob( Configuration conf
+                               , String table
+                               , String cf
+                               , String accessTrackerTable
+                               , String accessTrackerColumnFamily
+                               , Long ts
+                               ) throws IOException
+    {
+        Job job = new Job(conf);
+        job.setJobName("LeastRecentlyUsedPruner: Pruning " +  table + ":" + cf + " since " + new SimpleDateFormat().format(new Date(ts)));
+        System.out.println("Configuring " + job.getJobName());
+        job.setJarByClass(LeastRecentlyUsedPruner.class);
+        job.getConfiguration().setLong(PrunerMapper.TIMESTAMP_CONF, ts);
+        job.getConfiguration().set(PrunerMapper.ACCESS_TRACKER_NAME_CONF, table);
+        job.getConfiguration().set(PrunerMapper.ACCESS_TRACKER_CF_CONF, accessTrackerColumnFamily);
+        job.getConfiguration().set(PrunerMapper.ACCESS_TRACKER_TABLE_CONF, accessTrackerTable);
+        setupHBaseJob(job, table, cf);
+        job.setNumReduceTasks(0);
+        return job;
+    }
+
+    public static void main(String... argv) throws IOException, java.text.ParseException, ClassNotFoundException, InterruptedException {
+        Configuration conf = HBaseConfiguration.create();
+        String[] otherArgs = new GenericOptionsParser(conf, argv).getRemainingArgs();
+
+        CommandLine cli = BulkLoadOptions.parse(new PosixParser(), otherArgs);
+        Long ts = BulkLoadOptions.getTimestamp(cli);
+        String table = BulkLoadOptions.TABLE.get(cli);
+        String cf = BulkLoadOptions.COLUMN_FAMILY.get(cli);
+        String accessTrackerTable = BulkLoadOptions.ACCESS_TABLE.get(cli);
+        String accessTrackerCF = BulkLoadOptions.ACCESS_COLUMN_FAMILY.get(cli);
+        Job job = createJob(conf, table, cf, accessTrackerTable, accessTrackerCF, ts);
+        System.exit(job.waitForCompletion(true) ? 0 : 1);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
new file mode 100644
index 0000000..7d7ef98
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/bulk/ThreatIntelBulkLoader.java
@@ -0,0 +1,230 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.dataloads.bulk;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.io.Files;
+import org.apache.commons.cli.*;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.metron.dataloads.extractor.ExtractorHandler;
+import org.apache.metron.dataloads.hbase.mr.BulkLoadMapper;
+import org.apache.metron.hbase.converters.HbaseConverter;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelConverter;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.text.*;
+import java.util.Date;
+
+public class ThreatIntelBulkLoader  {
+    private static abstract class OptionHandler implements Function<String, Option> {}
+    private enum BulkLoadOptions {
+        HELP("h", new OptionHandler() {
+
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                return new Option(s, "help", false, "Generate Help screen");
+            }
+        })
+        ,TABLE("t", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "table", true, "HBase table to import data into");
+                o.setRequired(true);
+                o.setArgName("HBASE_TABLE");
+                return o;
+            }
+        })
+        ,COLUMN_FAMILY("f", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "column_family", true, "Column family of the HBase table to import into");
+                o.setRequired(true);
+                o.setArgName("CF_NAME");
+                return o;
+            }
+        })
+        ,EXTRACTOR_CONFIG("e", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "extractor_config", true, "JSON Document describing the extractor for this input data source");
+                o.setArgName("JSON_FILE");
+                o.setRequired(true);
+                return o;
+            }
+        })
+        ,INPUT_DATA("i", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "input", true, "Input directory in HDFS for the data to import into HBase");
+                o.setArgName("DIR");
+                o.setRequired(true);
+                return o;
+            }
+        })
+        ,AS_OF_TIME("a", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "as_of", true, "The last read timestamp to mark the records with (omit for time of execution)");
+                o.setArgName("datetime");
+                o.setRequired(false);
+                return o;
+            }
+        })
+        ,AS_OF_TIME_FORMAT("t", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "as_of_format", true, "The format of the as_of time (only used in conjunction with the as_of option)");
+                o.setArgName("format");
+                o.setRequired(false);
+                return o;
+            }
+        })
+        ,CONVERTER("c", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "converter", true, "The HBase converter class to use (Default is threat intel)");
+                o.setArgName("class");
+                o.setRequired(false);
+                return o;
+            }
+        })
+        ;
+        Option option;
+        String shortCode;
+        BulkLoadOptions(String shortCode, OptionHandler optionHandler) {
+            this.shortCode = shortCode;
+            this.option = optionHandler.apply(shortCode);
+        }
+
+        public boolean has(CommandLine cli) {
+            return cli.hasOption(shortCode);
+        }
+
+        public String get(CommandLine cli) {
+            return cli.getOptionValue(shortCode);
+        }
+
+        public static CommandLine parse(CommandLineParser parser, String[] args) {
+            try {
+                CommandLine cli = parser.parse(getOptions(), args);
+                if(ThreatIntelBulkLoader.BulkLoadOptions.HELP.has(cli)) {
+                    printHelp();
+                    System.exit(0);
+                }
+                return cli;
+            } catch (ParseException e) {
+                System.err.println("Unable to parse args: " + Joiner.on(' ').join(args));
+                e.printStackTrace(System.err);
+                printHelp();
+                System.exit(-1);
+                return null;
+            }
+        }
+
+        public static void printHelp() {
+            HelpFormatter formatter = new HelpFormatter();
+            formatter.printHelp( "ThreatIntelBulkLoader", getOptions());
+        }
+
+        public static Options getOptions() {
+            Options ret = new Options();
+            for(BulkLoadOptions o : BulkLoadOptions.values()) {
+               ret.addOption(o.option);
+            }
+            return ret;
+        }
+    }
+    private static long getTimestamp(CommandLine cli) throws java.text.ParseException {
+        if(BulkLoadOptions.AS_OF_TIME.has(cli)) {
+            if(!BulkLoadOptions.AS_OF_TIME_FORMAT.has(cli)) {
+                throw new IllegalStateException("Unable to proceed: Specified as_of_time without an associated format.");
+            }
+            else {
+                DateFormat format = new SimpleDateFormat(BulkLoadOptions.AS_OF_TIME_FORMAT.get(cli));
+                Date d = format.parse(BulkLoadOptions.AS_OF_TIME.get(cli));
+                return d.getTime();
+            }
+        }
+        else {
+            return System.currentTimeMillis();
+        }
+    }
+    private static String readExtractorConfig(File configFile) throws IOException {
+        return Joiner.on("\n").join(Files.readLines(configFile, Charset.defaultCharset()));
+    }
+
+    public static Job createJob(Configuration conf, String input, String table, String cf, String extractorConfigContents, long ts, HbaseConverter converter) throws IOException {
+        Job job = new Job(conf);
+        job.setJobName("ThreatIntelBulkLoader: " + input + " => " +  table + ":" + cf);
+        System.out.println("Configuring " + job.getJobName());
+        job.setJarByClass(ThreatIntelBulkLoader.class);
+        job.setMapperClass(org.apache.metron.dataloads.hbase.mr.BulkLoadMapper.class);
+        job.setOutputFormatClass(TableOutputFormat.class);
+        job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, table);
+        job.getConfiguration().set(BulkLoadMapper.COLUMN_FAMILY_KEY, cf);
+        job.getConfiguration().set(BulkLoadMapper.CONFIG_KEY, extractorConfigContents);
+        job.getConfiguration().set(BulkLoadMapper.LAST_SEEN_KEY, "" + ts);
+        job.getConfiguration().set(BulkLoadMapper.CONVERTER_KEY, converter.getClass().getName());
+        job.setOutputKeyClass(ImmutableBytesWritable.class);
+        job.setOutputValueClass(Put.class);
+        job.setNumReduceTasks(0);
+        ExtractorHandler handler = ExtractorHandler.load(extractorConfigContents);
+        handler.getInputFormatHandler().set(job, new Path(input), handler.getConfig());
+        return job;
+    }
+
+    public static void main(String... argv) throws IOException, java.text.ParseException, ClassNotFoundException, InterruptedException, IllegalAccessException, InstantiationException {
+        Configuration conf = HBaseConfiguration.create();
+        String[] otherArgs = new GenericOptionsParser(conf, argv).getRemainingArgs();
+
+        CommandLine cli = BulkLoadOptions.parse(new PosixParser(), otherArgs);
+        Long ts = getTimestamp(cli);
+        String input = BulkLoadOptions.INPUT_DATA.get(cli);
+        String table = BulkLoadOptions.TABLE.get(cli);
+        String cf = BulkLoadOptions.COLUMN_FAMILY.get(cli);
+        String extractorConfigContents = readExtractorConfig(new File(BulkLoadOptions.EXTRACTOR_CONFIG.get(cli)));
+        String converterClass = ThreatIntelConverter.class.getName();
+        if(BulkLoadOptions.CONVERTER.has(cli)) {
+            converterClass = BulkLoadOptions.CONVERTER.get(cli);
+        }
+        HbaseConverter converter = (HbaseConverter) Class.forName(converterClass).newInstance();
+        Job job = createJob(conf, input, table, cf, extractorConfigContents, ts, converter);
+        System.out.println(conf);
+        System.exit(job.waitForCompletion(true) ? 0 : 1);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/Extractor.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/Extractor.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/Extractor.java
index 41f667b..30e56d8 100644
--- a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/Extractor.java
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/Extractor.java
@@ -17,12 +17,14 @@
  */
 package org.apache.metron.dataloads.extractor;
 
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.metron.reference.lookup.LookupKV;
 import org.apache.metron.threatintel.ThreatIntelResults;
 
 import java.io.IOException;
 import java.util.Map;
 
 public interface Extractor {
-    Iterable<ThreatIntelResults> extract(String line) throws IOException;
+    Iterable<LookupKV> extract(String line) throws IOException;
     void initialize(Map<String, Object> config);
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java
index f5ec434..94e9ccb 100644
--- a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/csv/CSVExtractor.java
@@ -21,7 +21,12 @@ import com.google.common.base.Splitter;
 import com.google.common.collect.Iterables;
 import com.opencsv.CSVParser;
 import com.opencsv.CSVParserBuilder;
+import org.apache.hadoop.hbase.client.Put;
 import org.apache.metron.dataloads.extractor.Extractor;
+import org.apache.metron.hbase.converters.HbaseConverter;
+import org.apache.metron.reference.lookup.LookupKV;
+import org.apache.metron.reference.lookup.LookupKey;
+import org.apache.metron.reference.lookup.LookupValue;
 import org.apache.metron.threatintel.ThreatIntelResults;
 
 import java.io.IOException;
@@ -31,24 +36,26 @@ public class CSVExtractor implements Extractor {
     public static final String COLUMNS_KEY="columns";
     public static final String INDICATOR_COLUMN_KEY="indicator_column";
     public static final String SEPARATOR_KEY="separator";
+    public static final String LOOKUP_CONVERTER = "lookupConverter";
 
     private int indicatorColumn;
     private Map<String, Integer> columnMap = new HashMap<>();
     private CSVParser parser;
+    private LookupConverter converter = LookupConverters.THREAT_INTEL.getConverter();
 
     @Override
-    public Iterable<ThreatIntelResults> extract(String line) throws IOException {
+    public Iterable<LookupKV> extract(String line) throws IOException {
         if(line.trim().startsWith("#")) {
             //comment
             return Collections.emptyList();
         }
-        ThreatIntelResults ret = new ThreatIntelResults();
         String[] tokens = parser.parseLine(line);
-        ret.getKey().indicator = tokens[indicatorColumn];
+        LookupKey key = converter.toKey(tokens[indicatorColumn]);
+        Map<String, String> values = new HashMap<>();
         for(Map.Entry<String, Integer> kv : columnMap.entrySet()) {
-            ret.getValue().put(kv.getKey(), tokens[kv.getValue()]);
+            values.put(kv.getKey(), tokens[kv.getValue()]);
         }
-        return Arrays.asList(ret);
+        return Arrays.asList(new LookupKV(key, converter.toValue(values)));
     }
 
     private static Map.Entry<String, Integer> getColumnMapEntry(String column, int i) {
@@ -109,5 +116,8 @@ public class CSVExtractor implements Extractor {
             parser = new CSVParserBuilder().withSeparator(separator)
                                            .build();
         }
+        if(config.containsKey(LOOKUP_CONVERTER)) {
+           converter = LookupConverters.getConverter((String) config.get(LOOKUP_CONVERTER));
+        }
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/csv/LookupConverter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/csv/LookupConverter.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/csv/LookupConverter.java
new file mode 100644
index 0000000..9e9b79f
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/csv/LookupConverter.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.dataloads.extractor.csv;
+
+import org.apache.metron.hbase.converters.HbaseConverter;
+import org.apache.metron.reference.lookup.LookupKey;
+import org.apache.metron.reference.lookup.LookupValue;
+
+import java.util.Map;
+
+public interface LookupConverter {
+    LookupKey toKey(String indicator);
+    LookupValue toValue(Map<String, String> metadata);
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/csv/LookupConverters.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/csv/LookupConverters.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/csv/LookupConverters.java
new file mode 100644
index 0000000..7f9218a
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/csv/LookupConverters.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.dataloads.extractor.csv;
+
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
+import org.apache.metron.reference.lookup.LookupKey;
+import org.apache.metron.reference.lookup.LookupValue;
+
+import java.util.Map;
+
+public enum LookupConverters {
+
+    THREAT_INTEL(new LookupConverter() {
+        @Override
+        public LookupKey toKey(String indicator) {
+            return new ThreatIntelKey(indicator);
+        }
+
+        @Override
+        public LookupValue toValue(Map<String, String> metadata) {
+            return new ThreatIntelValue(metadata);
+        }
+    })
+    ;
+    LookupConverter converter;
+    LookupConverters(LookupConverter converter) {
+        this.converter = converter;
+    }
+    public LookupConverter getConverter() {
+        return converter;
+    }
+
+    public static LookupConverter getConverter(String name) {
+        try {
+            return LookupConverters.valueOf(name).getConverter();
+        }
+        catch(Throwable t) {
+            try {
+                return (LookupConverter) Class.forName(name).newInstance();
+            } catch (InstantiationException e) {
+                throw new IllegalStateException("Unable to parse " + name, e);
+            } catch (IllegalAccessException e) {
+                throw new IllegalStateException("Unable to parse " + name, e);
+            } catch (ClassNotFoundException e) {
+                throw new IllegalStateException("Unable to parse " + name, e);
+            }
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java
index 039092f..b3829b4 100644
--- a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java
@@ -23,9 +23,12 @@ import org.apache.commons.io.FileUtils;
 import org.apache.metron.dataloads.extractor.Extractor;
 import org.apache.metron.dataloads.extractor.stix.types.ObjectTypeHandler;
 import org.apache.metron.dataloads.extractor.stix.types.ObjectTypeHandlers;
+import org.apache.metron.reference.lookup.LookupKV;
 import org.apache.metron.threatintel.ThreatIntelResults;
 import org.mitre.cybox.common_2.*;
 import org.mitre.cybox.cybox_2.ObjectType;
+import org.mitre.cybox.cybox_2.Observable;
+import org.mitre.cybox.cybox_2.Observables;
 import org.mitre.stix.common_1.IndicatorBaseType;
 import org.mitre.stix.indicator_2.Indicator;
 import org.mitre.stix.stix_1.STIXPackage;
@@ -39,9 +42,35 @@ import java.util.Map;
 public class StixExtractor implements Extractor {
     Map<String, Object> config;
     @Override
-    public Iterable<ThreatIntelResults> extract(String line) throws IOException {
-        STIXPackage stixPackage = STIXPackage.fromXMLString(line);
-        List<ThreatIntelResults> ret = new ArrayList<>();
+    public Iterable<LookupKV> extract(String line) throws IOException {
+        STIXPackage stixPackage = STIXPackage.fromXMLString(line.replaceAll("\"Equal\"", "\"Equals\""));
+        List<LookupKV> ret = new ArrayList<>();
+        for(Observable o : getObservables(stixPackage)) {
+            ObjectType obj = o.getObject();
+            if(obj != null) {
+                ObjectPropertiesType props = obj.getProperties();
+                if(props != null) {
+                    ObjectTypeHandler handler = ObjectTypeHandlers.getHandlerByInstance(props);
+                    if (handler != null) {
+                        Iterable<LookupKV> extractions = handler.extract(props, config);
+                        for(LookupKV extraction : extractions) {
+                            ret.add(extraction);
+                        }
+                    }
+                }
+            }
+        }
+        return ret;
+    }
+
+    public List<Observable> getObservables(STIXPackage stixPackage) {
+        List<Observable> ret = new ArrayList<>();
+        Observables observables = stixPackage.getObservables();
+        if(observables != null) {
+            for (Observable o : observables.getObservables()) {
+                ret.add(o);
+            }
+        }
         if (stixPackage.getIndicators() != null) {
             if (stixPackage.getIndicators().getIndicators() != null) {
                 List<IndicatorBaseType> indicators = stixPackage.getIndicators().getIndicators();
@@ -49,12 +78,7 @@ public class StixExtractor implements Extractor {
                 for (int i = 0; i < indicatorCount; i++) {
                     Indicator indicator = (Indicator) indicators.get(i);
                     if (indicator.getObservable() != null) {
-                        ObjectType obj = indicator.getObservable().getObject();
-                        ObjectPropertiesType props = obj.getProperties();
-                        ObjectTypeHandler handler = ObjectTypeHandlers.getHandlerByInstance(props);
-                        if(handler != null) {
-                            Iterables.addAll(ret, handler.extract(props, config));
-                        }
+                        ret.add(indicator.getObservable());
                     }
                 }
             }
@@ -102,7 +126,7 @@ public class StixExtractor implements Extractor {
 
         String line = FileUtils.readFileToString(file);
         StixExtractor extractor = new StixExtractor();
-        for(ThreatIntelResults results : extractor.extract(line)) {
+        for(LookupKV results : extractor.extract(line)) {
             System.out.println(results);
         }
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/AddressHandler.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/AddressHandler.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/AddressHandler.java
index ee4ff78..638a9ce 100644
--- a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/AddressHandler.java
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/AddressHandler.java
@@ -19,10 +19,10 @@ package org.apache.metron.dataloads.extractor.stix.types;
 
 import com.google.common.base.Splitter;
 import org.apache.metron.dataloads.extractor.stix.StixExtractor;
-import org.apache.metron.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
+import org.apache.metron.reference.lookup.LookupKV;
 import org.apache.metron.threatintel.ThreatIntelResults;
-import org.mitre.cybox.common_2.ConditionApplicationEnum;
-import org.mitre.cybox.common_2.ConditionTypeEnum;
 import org.mitre.cybox.common_2.StringObjectPropertyType;
 import org.mitre.cybox.objects.Address;
 import org.mitre.cybox.objects.CategoryTypeEnum;
@@ -42,8 +42,8 @@ public class AddressHandler extends AbstractObjectTypeHandler<Address> {
     }
 
     @Override
-    public Iterable<ThreatIntelResults> extract(final Address type, Map<String, Object> config) throws IOException {
-        List<ThreatIntelResults> ret = new ArrayList<>();
+    public Iterable<LookupKV> extract(final Address type, Map<String, Object> config) throws IOException {
+        List<LookupKV> ret = new ArrayList<>();
         final CategoryTypeEnum category= type.getCategory();
         if(!SUPPORTED_CATEGORIES.contains(category)) {
            return ret;
@@ -61,13 +61,15 @@ public class AddressHandler extends AbstractObjectTypeHandler<Address> {
         }
         StringObjectPropertyType value = type.getAddressValue();
         for(String token : StixExtractor.split(value)) {
-            ThreatIntelResults results = new ThreatIntelResults(new ThreatIntelKey(token),
+            LookupKV results = new LookupKV(new ThreatIntelKey(token)
+                                           , new ThreatIntelValue(
                                                                     new HashMap<String, String>() {{
                                                                         put("source-type", "STIX");
-                                                                        put("indicator-type", "Address");
+                                                                        put("indicator-type", type.getClass().getSimpleName() + ":" + category);
                                                                         put("source", type.toXMLString());
                                                                     }}
-                                                                   );
+                                                                 )
+                                           );
                 ret.add(results);
         }
         return ret;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/DomainHandler.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/DomainHandler.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/DomainHandler.java
index e0444fb..233550b 100644
--- a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/DomainHandler.java
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/DomainHandler.java
@@ -18,7 +18,9 @@
 package org.apache.metron.dataloads.extractor.stix.types;
 
 import org.apache.metron.dataloads.extractor.stix.StixExtractor;
-import org.apache.metron.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
+import org.apache.metron.reference.lookup.LookupKV;
 import org.apache.metron.threatintel.ThreatIntelResults;
 import org.mitre.cybox.common_2.StringObjectPropertyType;
 import org.mitre.cybox.objects.DomainName;
@@ -34,19 +36,21 @@ public class DomainHandler extends AbstractObjectTypeHandler<DomainName> {
     }
 
     @Override
-    public Iterable<ThreatIntelResults> extract(final DomainName type, Map<String, Object> config) throws IOException {
-        List<ThreatIntelResults> ret = new ArrayList<>();
+    public Iterable<LookupKV> extract(final DomainName type, Map<String, Object> config) throws IOException {
+        List<LookupKV> ret = new ArrayList<>();
         final DomainNameTypeEnum domainType = type.getType();
-        if(SUPPORTED_TYPES.contains(domainType)) {
+        if(domainType == null || SUPPORTED_TYPES.contains(domainType)) {
             StringObjectPropertyType value = type.getValue();
             for (String token : StixExtractor.split(value)) {
-                ThreatIntelResults results = new ThreatIntelResults(new ThreatIntelKey(token),
-                        new HashMap<String, String>() {{
-                            put("source-type", "STIX");
-                            put("indicator-type", "DomainName");
-                            put("source", type.toXMLString());
-                        }}
-                );
+                LookupKV results = new LookupKV(new ThreatIntelKey(token)
+                                               , new ThreatIntelValue(
+                                                    new HashMap<String, String>() {{
+                                                        put("source-type", "STIX");
+                                                        put("indicator-type", type.getClass().getSimpleName() + ":" + DomainNameTypeEnum.FQDN);
+                                                        put("source", type.toXMLString());
+                                                    }}
+                                                                    )
+                                               );
                 ret.add(results);
             }
         }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/HostnameHandler.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/HostnameHandler.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/HostnameHandler.java
index d0c55a9..702c440 100644
--- a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/HostnameHandler.java
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/HostnameHandler.java
@@ -19,7 +19,9 @@
 package org.apache.metron.dataloads.extractor.stix.types;
 
 import org.apache.metron.dataloads.extractor.stix.StixExtractor;
-import org.apache.metron.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
+import org.apache.metron.reference.lookup.LookupKV;
 import org.apache.metron.threatintel.ThreatIntelResults;
 import org.mitre.cybox.common_2.StringObjectPropertyType;
 import org.mitre.cybox.objects.Hostname;
@@ -36,17 +38,18 @@ public class HostnameHandler  extends AbstractObjectTypeHandler<Hostname>{
     }
 
     @Override
-    public Iterable<ThreatIntelResults> extract(final Hostname type, Map<String, Object> config) throws IOException {
+    public Iterable<LookupKV> extract(final Hostname type, Map<String, Object> config) throws IOException {
         StringObjectPropertyType value = type.getHostnameValue();
-        List<ThreatIntelResults> ret = new ArrayList<>();
+        List<LookupKV> ret = new ArrayList<>();
         for(String token : StixExtractor.split(value)) {
-            ThreatIntelResults results = new ThreatIntelResults(new ThreatIntelKey(token),
-                                                                    new HashMap<String, String>() {{
+            LookupKV results = new LookupKV(new ThreatIntelKey(token)
+                                           , new ThreatIntelValue(new HashMap<String, String>() {{
                                                                         put("source-type", "STIX");
-                                                                        put("indicator-type", "Hostname");
+                                                                        put("indicator-type", type.getClass().getSimpleName());
                                                                         put("source", type.toXMLString());
                                                                     }}
-                                                                   );
+                                                                 )
+                                           );
                 ret.add(results);
         }
         return ret;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandler.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandler.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandler.java
index f639be0..e5a5296 100644
--- a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandler.java
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandler.java
@@ -18,6 +18,7 @@
 package org.apache.metron.dataloads.extractor.stix.types;
 
 import org.apache.metron.dataloads.extractor.Extractor;
+import org.apache.metron.reference.lookup.LookupKV;
 import org.apache.metron.threatintel.ThreatIntelResults;
 import org.mitre.cybox.common_2.ObjectPropertiesType;
 
@@ -25,6 +26,6 @@ import java.io.IOException;
 import java.util.Map;
 
 public interface ObjectTypeHandler<T extends ObjectPropertiesType> {
-    Iterable<ThreatIntelResults> extract(T type, Map<String, Object> config) throws IOException;
+    Iterable<LookupKV> extract(T type, Map<String, Object> config) throws IOException;
     Class<T> getTypeClass();
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapper.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapper.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapper.java
index 0ad09e6..04714d9 100644
--- a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapper.java
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapper.java
@@ -24,8 +24,10 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.metron.dataloads.extractor.Extractor;
 import org.apache.metron.dataloads.extractor.ExtractorHandler;
+import org.apache.metron.hbase.converters.HbaseConverter;
+import org.apache.metron.reference.lookup.LookupKV;
 import org.apache.metron.threatintel.ThreatIntelResults;
-import org.apache.metron.threatintel.hbase.Converter;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelConverter;
 
 import java.io.IOException;
 
@@ -34,9 +36,11 @@ public class BulkLoadMapper extends Mapper<Object, Text, ImmutableBytesWritable,
     public static final String CONFIG_KEY="bl_extractor_config";
     public static final String COLUMN_FAMILY_KEY = "bl_column_family";
     public static final String LAST_SEEN_KEY = "bl_last_seen";
+    public static final String CONVERTER_KEY = "bl_converter";
     Extractor extractor = null;
     String columnFamily = null;
     Long lastSeen = null;
+    HbaseConverter converter;
     @Override
     public void setup(Context context) throws IOException,
             InterruptedException {
@@ -45,19 +49,28 @@ public class BulkLoadMapper extends Mapper<Object, Text, ImmutableBytesWritable,
 
     @Override
     public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
-        for(ThreatIntelResults results : extractor.extract(value.toString())) {
+        for(LookupKV results : extractor.extract(value.toString())) {
             if (results != null) {
-                Put put = Converter.INSTANCE.toPut(columnFamily, results.getKey(), results.getValue(), lastSeen);
+                Put put = converter.toPut(columnFamily, results.getKey(), results.getValue());
                 write(new ImmutableBytesWritable(results.getKey().toBytes()), put, context);
             }
         }
     }
 
-    protected void initialize(Configuration configuration) throws IOException {
+    protected void initialize(Configuration configuration) throws IOException{
         String configStr = configuration.get(CONFIG_KEY);
         extractor = ExtractorHandler.load(configStr).getExtractor();
         columnFamily = configuration.get(COLUMN_FAMILY_KEY);
         lastSeen = Long.parseLong(configuration.get(LAST_SEEN_KEY));
+        try {
+            converter = (HbaseConverter) Class.forName(configuration.get(CONVERTER_KEY)).newInstance();
+        } catch (InstantiationException e) {
+            throw new IllegalStateException("Unable to create converter object: " + configuration.get(CONVERTER_KEY), e);
+        } catch (IllegalAccessException e) {
+            throw new IllegalStateException("Unable to create converter object: " + configuration.get(CONVERTER_KEY), e);
+        } catch (ClassNotFoundException e) {
+            throw new IllegalStateException("Unable to create converter object: " + configuration.get(CONVERTER_KEY), e);
+        }
     }
 
     protected void write(ImmutableBytesWritable key, Put value, Context context) throws IOException, InterruptedException {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/hbase/mr/PrunerMapper.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/hbase/mr/PrunerMapper.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/hbase/mr/PrunerMapper.java
index 3e19b07..bf33eed 100644
--- a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/hbase/mr/PrunerMapper.java
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/hbase/mr/PrunerMapper.java
@@ -70,6 +70,11 @@ public class PrunerMapper extends TableMapper<ImmutableBytesWritable, Delete> {
             public byte[] toBytes() {
                 return bytes;
             }
+
+            @Override
+            public void fromBytes(byte[] in) {
+
+            }
         };
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/ConnectionType.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/ConnectionType.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/ConnectionType.java
new file mode 100644
index 0000000..1e95507
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/ConnectionType.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.dataloads.taxii;
+
+public enum ConnectionType {
+   POLL, DISCOVER;
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TableInfo.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TableInfo.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TableInfo.java
new file mode 100644
index 0000000..ddf542e
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TableInfo.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.dataloads.taxii;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+import org.apache.hadoop.hbase.client.HTableInterface;
+
+public class TableInfo {
+    private String tableName;
+    private String columnFamily;
+    public TableInfo(String s) {
+        Iterable<String> i = Splitter.on(":").split(s);
+        if(Iterables.size(i) != 2) {
+            throw new IllegalStateException("Malformed table:cf => " + s);
+        }
+        tableName = Iterables.getFirst(i, null);
+        columnFamily = Iterables.getLast(i);
+    }
+
+    public String getTableName() {
+        return tableName;
+    }
+
+    public String getColumnFamily() {
+        return columnFamily;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        TableInfo tableInfo = (TableInfo) o;
+
+        if (getTableName() != null ? !getTableName().equals(tableInfo.getTableName()) : tableInfo.getTableName() != null)
+            return false;
+        return getColumnFamily() != null ? getColumnFamily().equals(tableInfo.getColumnFamily()) : tableInfo.getColumnFamily() == null;
+
+    }
+
+    @Override
+    public int hashCode() {
+        int result = getTableName() != null ? getTableName().hashCode() : 0;
+        result = 31 * result + (getColumnFamily() != null ? getColumnFamily().hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "TableInfo{" +
+                "tableName='" + tableName + '\'' +
+                ", columnFamily='" + columnFamily + '\'' +
+                '}';
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TaxiiConnectionConfig.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TaxiiConnectionConfig.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TaxiiConnectionConfig.java
new file mode 100644
index 0000000..dab8f0c
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TaxiiConnectionConfig.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.dataloads.taxii;
+
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TaxiiConnectionConfig {
+    final static ObjectMapper _mapper = new ObjectMapper();
+    private URL endpoint;
+    private int port = 443;
+    private URL proxy;
+    private String username;
+    private String password;
+    private ConnectionType type;
+    private String collection = "default";
+    private String subscriptionId = null;
+    private Date beginTime;
+    private Map<String, TableInfo> tableMap;
+    public TaxiiConnectionConfig withTableMap(Map<String, TableInfo> tableMap) {
+        this.tableMap = tableMap;
+        return this;
+    }
+    public TaxiiConnectionConfig withBeginTime(Date time) {
+        this.beginTime = time;
+        return this;
+    }
+    public TaxiiConnectionConfig withSubscriptionId(String subId) {
+        this.subscriptionId = subId;
+        return this;
+    }
+    public TaxiiConnectionConfig withCollection(String collection) {
+        this.collection = collection;
+        return this;
+    }
+
+    public TaxiiConnectionConfig withPort(int port) {
+        this.port = port;
+        return this;
+    }
+    public TaxiiConnectionConfig withEndpoint(URL endpoint) {
+        this.endpoint = endpoint;
+        return this;
+    }
+    public TaxiiConnectionConfig withProxy(URL proxy) {
+        this.proxy = proxy;
+        return this;
+    }
+    public TaxiiConnectionConfig withUsername(String username) {
+        this.username = username;
+        return this;
+    }
+    public TaxiiConnectionConfig withPassword(String password) {
+        this.password = password;
+        return this;
+    }
+    public TaxiiConnectionConfig withConnectionType(ConnectionType type) {
+        this.type= type;
+        return this;
+    }
+
+    public void setEndpoint(String endpoint) throws MalformedURLException {
+        this.endpoint = new URL(endpoint);
+    }
+
+    public void setPort(int port) {
+        this.port = port;
+    }
+
+    public void setProxy(String proxy) throws MalformedURLException {
+        this.proxy = new URL(proxy);
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public void setType(ConnectionType type) {
+        this.type = type;
+    }
+
+    public void setCollection(String collection) {
+        this.collection = collection;
+    }
+
+    public void setSubscriptionId(String subscriptionId) {
+        this.subscriptionId = subscriptionId;
+    }
+
+    public void setBeginTime(String beginTime) throws ParseException {
+        SimpleDateFormat sdf = (SimpleDateFormat)DateFormat.getDateInstance(DateFormat.MEDIUM);
+        this.beginTime = sdf.parse(beginTime);
+    }
+
+    public void setTableMap(Map<String, String> tableMap) {
+        this.tableMap = new HashMap<>();
+        for(Map.Entry<String, String> kv : tableMap.entrySet()) {
+            this.tableMap.put(kv.getKey(), new TableInfo(kv.getValue()));
+        }
+    }
+
+    public Map<String, TableInfo> getTableMap() {
+        return tableMap;
+    }
+
+    public Date getBeginTime() {
+        return beginTime;
+    }
+    public int getPort() {
+        return port;
+    }
+    public URL getEndpoint() {
+        return endpoint;
+    }
+
+    public URL getProxy() {
+        return proxy;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public ConnectionType getType() {
+        return type;
+    }
+
+    public String getCollection() {
+        return collection;
+    }
+    public String getSubscriptionId() {
+        return subscriptionId;
+    }
+    public static synchronized TaxiiConnectionConfig load(InputStream is) throws IOException {
+        TaxiiConnectionConfig ret = _mapper.readValue(is, TaxiiConnectionConfig.class);
+        return ret;
+    }
+    public static synchronized TaxiiConnectionConfig load(String s, Charset c) throws IOException {
+        return load( new ByteArrayInputStream(s.getBytes(c)));
+    }
+    public static synchronized TaxiiConnectionConfig load(String s) throws IOException {
+        return load( s, Charset.defaultCharset());
+    }
+
+    @Override
+    public String toString() {
+        return "TaxiiConnectionConfig{" +
+                "endpoint=" + endpoint +
+                ", port=" + port +
+                ", proxy=" + proxy +
+                ", username='" + username + '\'' +
+                ", password=" + (password == null?"null" : "'******'") +
+                ", type=" + type +
+                ", collection='" + collection + '\'' +
+                ", subscriptionId='" + subscriptionId + '\'' +
+                ", beginTime=" + beginTime +
+                ", tableMap=" + tableMap +
+                '}';
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TaxiiHandler.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TaxiiHandler.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TaxiiHandler.java
new file mode 100644
index 0000000..614adec
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TaxiiHandler.java
@@ -0,0 +1,403 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.dataloads.taxii;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.AuthCache;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLContextBuilder;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.BasicAuthCache;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.log4j.Logger;
+import org.apache.metron.dataloads.extractor.Extractor;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelConverter;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
+import org.apache.metron.reference.lookup.LookupKV;
+import org.mitre.taxii.client.HttpClient;
+import org.mitre.taxii.messages.xml11.*;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import javax.xml.bind.JAXBException;
+import javax.xml.datatype.DatatypeConfigurationException;
+import javax.xml.datatype.DatatypeFactory;
+import javax.xml.datatype.XMLGregorianCalendar;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import java.io.*;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+public class TaxiiHandler extends TimerTask {
+    private static final Logger LOG = Logger.getLogger(TaxiiHandler.class);
+
+    private static ThreadLocal<TaxiiXmlFactory> xmlFactory = new ThreadLocal<TaxiiXmlFactory>() {
+        @Override
+        protected TaxiiXmlFactory initialValue() {
+            return new TaxiiXmlFactory();
+        }
+    };
+    private static ThreadLocal<ObjectFactory> messageFactory = new ThreadLocal<ObjectFactory>() {
+        @Override
+        protected ObjectFactory initialValue() {
+            return new ObjectFactory();
+        }
+    };
+
+    private HttpClient taxiiClient;
+    private URL endpoint;
+    private Extractor extractor;
+    private Map<String, TableInfo> tableMap;
+    private Map<TableInfo, HTableInterface> connectionCache = new HashMap<>();
+    private HttpClientContext context;
+    private String collection;
+    private String subscriptionId;
+    private ThreatIntelConverter converter = new ThreatIntelConverter();
+    private Date beginTime;
+    private Configuration config;
+    private boolean inProgress = false;
+    public TaxiiHandler( TaxiiConnectionConfig connectionConfig
+                       , Extractor extractor
+                       , Configuration config
+                       ) throws Exception
+    {
+        LOG.info("Loading configuration: " + connectionConfig);
+        this.extractor = extractor;
+        this.collection = connectionConfig.getCollection();
+        this.subscriptionId = connectionConfig.getSubscriptionId();
+        this.tableMap = connectionConfig.getTableMap();
+        this.beginTime = connectionConfig.getBeginTime();
+        this.config = config;
+        initializeClient(connectionConfig);
+        LOG.info("Configured, starting polling " + endpoint + " for " + collection);
+    }
+
+    protected synchronized HTableInterface getTable(TableInfo tableInfo) throws IOException {
+        HTableInterface ret = connectionCache.get(tableInfo);
+        if(ret == null) {
+            ret = createHTable(tableInfo);
+            connectionCache.put(tableInfo, ret);
+        }
+        return ret;
+    }
+
+    protected synchronized HTableInterface createHTable(TableInfo tableInfo) throws IOException {
+        return new HTable(config, tableInfo.getTableName());
+    }
+    /**
+     * The action to be performed by this timer task.
+     */
+    @Override
+    public void run() {
+        if(inProgress) {
+            return;
+        }
+        Date ts = new Date();
+        LOG.info("Polling..." + new SimpleDateFormat().format(ts));
+        try {
+            inProgress = true;
+            // Prepare the message to send.
+            String sessionID = MessageHelper.generateMessageId();
+            PollRequest request = messageFactory.get().createPollRequest()
+                    .withMessageId(sessionID)
+                    .withCollectionName(collection);
+            if (subscriptionId != null) {
+                request = request.withSubscriptionID(subscriptionId);
+            } else {
+                request = request.withPollParameters(messageFactory.get().createPollParametersType());
+            }
+            if (beginTime != null) {
+                Calendar gc = GregorianCalendar.getInstance();
+                gc.setTime(beginTime);
+                XMLGregorianCalendar gTime = null;
+                try {
+                    gTime = DatatypeFactory.newInstance().newXMLGregorianCalendar((GregorianCalendar) gc).normalize();
+                } catch (DatatypeConfigurationException e) {
+                    LOG.error("Unable to set the begin time", e);
+                }
+                gTime.setFractionalSecond(null);
+                LOG.info("Begin Time: " + gTime);
+                request.setExclusiveBeginTimestamp(gTime);
+            }
+
+            try {
+                PollResponse response = call(request, PollResponse.class);
+                LOG.info("Got Poll Response with " + response.getContentBlocks().size() + " blocks");
+                int numProcessed = 0;
+                long avgTimeMS = 0;
+                long timeStartedBlock = System.currentTimeMillis();
+                for (ContentBlock block : response.getContentBlocks()) {
+                    AnyMixedContentType content = block.getContent();
+                    for (Object o : content.getContent()) {
+                        numProcessed++;
+                        long timeS = System.currentTimeMillis();
+                        String xml = null;
+                        if (o instanceof Element) {
+                            Element element = (Element) o;
+                            xml = getStringFromDocument(element.getOwnerDocument());
+                            if(LOG.isDebugEnabled() && Math.random() < 0.01) {
+                                LOG.debug("Random Stix doc: " + xml);
+                            }
+                            for (LookupKV<ThreatIntelKey, ThreatIntelValue> kv : extractor.extract(xml)) {
+                                String indicatorType = kv.getValue().getMetadata().get("indicator-type");
+                                TableInfo tableInfo = tableMap.get(indicatorType);
+                                boolean persisted = false;
+                                if (tableInfo != null) {
+                                    kv.getValue().getMetadata().put("source_type", "taxii");
+                                    kv.getValue().getMetadata().put("taxii_url", endpoint.toString());
+                                    kv.getValue().getMetadata().put("taxii_collection", collection);
+                                    Put p = converter.toPut(tableInfo.getColumnFamily(), kv.getKey(), kv.getValue());
+                                    HTableInterface table = getTable(tableInfo);
+                                    table.put(p);
+                                    persisted = true;
+                                }
+                                LOG.info("Found Threat Intel: " + persisted + ", " + kv.getKey() + " => " + kv.getValue());
+                            }
+                        }
+                        avgTimeMS += System.currentTimeMillis() - timeS;
+                    }
+                    if( (numProcessed + 1) % 100 == 0) {
+                        LOG.info("Processed " + numProcessed + " in " + (System.currentTimeMillis() - timeStartedBlock) + " ms, avg time: " + avgTimeMS / content.getContent().size());
+                        timeStartedBlock = System.currentTimeMillis();
+                        avgTimeMS = 0;
+                        numProcessed = 0;
+                    }
+                }
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+                throw new RuntimeException("Unable to make request", e);
+            }
+        }
+        finally {
+            inProgress = false;
+            beginTime = ts;
+        }
+    }
+    public String getStringFromDocument(Document doc)
+    {
+        try
+        {
+            DOMSource domSource = new DOMSource(doc);
+            StringWriter writer = new StringWriter();
+            StreamResult result = new StreamResult(writer);
+            TransformerFactory tf = TransformerFactory.newInstance();
+            Transformer transformer = tf.newTransformer();
+            transformer.transform(domSource, result);
+            return writer.toString();
+        }
+        catch(TransformerException ex)
+        {
+            ex.printStackTrace();
+            return null;
+        }
+    }
+    private <RESPONSE_T> RESPONSE_T call( Object request, Class<RESPONSE_T> responseClazz) throws URISyntaxException, JAXBException, IOException {
+        return call(taxiiClient, endpoint.toURI(), request, context, responseClazz);
+    }
+
+    private void initializeClient(TaxiiConnectionConfig config) throws Exception {
+        LOG.info("Initializing client..");
+        if(context == null) {
+            context = createContext(config.getEndpoint(), config.getUsername(), config.getPassword(), config.getPort());
+        }
+        URL endpoint = config.getEndpoint();
+        if(config.getType() == ConnectionType.DISCOVER) {
+            LOG.info("Discovering endpoint");
+            endpoint = discoverPollingClient(config.getProxy(), endpoint, config.getUsername(), config.getPassword(), context, collection).pollEndpoint;
+            this.endpoint = endpoint;
+            LOG.info("Discovered endpoint as " + endpoint);
+        }
+        taxiiClient = buildClient(config.getProxy(), config.getUsername(), config.getPassword());
+    }
+
+    private static class DiscoveryResults {
+        URL pollEndpoint;
+        URL collectionManagementEndpoint;
+        List<String> collections = new ArrayList<>();
+    }
+    private static DiscoveryResults discoverPollingClient(URL proxy, URL endpoint, String username, String password, HttpClientContext context, String defaultCollection) throws Exception {
+
+        DiscoveryResults results = new DiscoveryResults();
+        {
+            HttpClient discoverClient = buildClient(proxy, username, password);
+            String sessionID = MessageHelper.generateMessageId();
+            // Prepare the message to send.
+            DiscoveryRequest request = messageFactory.get().createDiscoveryRequest()
+                    .withMessageId(sessionID);
+            DiscoveryResponse response = call(discoverClient, endpoint.toURI(), request, context, DiscoveryResponse.class);
+            for (ServiceInstanceType serviceInstance : response.getServiceInstances()) {
+                if (serviceInstance.isAvailable() && serviceInstance.getServiceType() == ServiceTypeEnum.POLL) {
+                    results.pollEndpoint = new URL(serviceInstance.getAddress());
+                }
+                else if(serviceInstance.isAvailable() && serviceInstance.getServiceType() == ServiceTypeEnum.COLLECTION_MANAGEMENT) {
+                    results.collectionManagementEndpoint= new URL(serviceInstance.getAddress());
+                }
+            }
+            if (results.pollEndpoint == null) {
+                throw new RuntimeException("Unable to discover a poll TAXII feed");
+            }
+        }
+        if(defaultCollection == null)
+        //get collections
+        {
+            HttpClient discoverClient = buildClient(proxy, username, password);
+            String sessionID = MessageHelper.generateMessageId();
+            CollectionInformationRequest request = messageFactory.get().createCollectionInformationRequest()
+                                                                 .withMessageId(sessionID);
+            CollectionInformationResponse response = call(discoverClient, results.collectionManagementEndpoint.toURI(), request, context, CollectionInformationResponse.class);
+            LOG.info("Unable to find the default collection; available collections are:");
+            for(CollectionRecordType c : response.getCollections()) {
+                LOG.info(c.getCollectionName());
+                results.collections.add(c.getCollectionName());
+            }
+            System.exit(0);
+        }
+        return results;
+    }
+
+    private static HttpClientContext createContext(URL endpoint, String username, String password, int port) {
+        HttpClientContext context = null;
+        HttpHost target = new HttpHost(endpoint.getHost(), port, endpoint.getProtocol());
+        if (username != null && password != null) {
+
+            CredentialsProvider credsProvider = new BasicCredentialsProvider();
+            credsProvider.setCredentials(
+                    new AuthScope(target.getHostName(), target.getPort()),
+                    new UsernamePasswordCredentials(username, password));
+
+            // http://hc.apache.org/httpcomponents-client-ga/tutorial/html/authentication.html
+            AuthCache authCache = new BasicAuthCache();
+            authCache.put(target, new BasicScheme());
+
+            // Add AuthCache to the execution context
+            context = HttpClientContext.create();
+            context.setCredentialsProvider(credsProvider);
+            context.setAuthCache(authCache);
+        } else {
+            context = null;
+        }
+        return context;
+    }
+
+
+    public static <RESPONSE_T, REQUEST_T> RESPONSE_T call( HttpClient taxiiClient
+            , URI endpoint
+            , REQUEST_T request
+            , HttpClientContext context
+            , Class<RESPONSE_T> responseClazz
+    ) throws JAXBException, IOException {
+        //TaxiiXml taxiiXml = xmlFactory.get().createTaxiiXml();
+        //String req = taxiiXml.marshalToString(request, true);
+        // Call the service
+        Object responseObj =  taxiiClient.callTaxiiService(endpoint, request, context);
+        LOG.info("Request made : " + request.getClass().getCanonicalName() + " => " + responseObj.getClass().getCanonicalName() + " (expected " + responseClazz.getCanonicalName() + ")");
+        //String resp = taxiiXml.marshalToString(responseObj, true);
+        try {
+            return responseClazz.cast(responseObj);
+        }
+        catch(ClassCastException cce) {
+            TaxiiXml taxiiXml = xmlFactory.get().createTaxiiXml();
+            String resp = taxiiXml.marshalToString(responseObj, true);
+            String msg = "Didn't return the response we expected: " + responseObj.getClass() + " \n" + resp;
+            LOG.error(msg, cce);
+            throw new RuntimeException(msg, cce);
+        }
+    }
+    private static HttpClient buildClient(URL proxy, String username, String password) throws Exception
+    {
+        HttpClient client = new HttpClient(); // Start with a default TAXII HTTP client.
+
+        // Create an Apache HttpClientBuilder to be customized by the command line arguments.
+        HttpClientBuilder builder = HttpClientBuilder.create().useSystemProperties();
+
+        // Proxy
+        if (proxy != null) {
+            HttpHost proxyHost = new HttpHost(proxy.getHost(), proxy.getPort(), proxy.getProtocol());
+            builder.setProxy(proxyHost);
+        }
+
+        // Basic authentication. User & Password
+        if (username != null ^ password != null) {
+            throw new Exception("'username' and 'password' arguments are required to appear together.");
+        }
+
+
+        // from:  http://stackoverflow.com/questions/19517538/ignoring-ssl-certificate-in-apache-httpclient-4-3
+        SSLContextBuilder ssbldr = new SSLContextBuilder();
+        ssbldr.loadTrustMaterial(null, new TrustSelfSignedStrategy());
+        SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(ssbldr.build(),SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
+
+
+        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
+                .register("http", new PlainConnectionSocketFactory())
+                .register("https", sslsf)
+                .build();
+
+
+        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
+        cm.setMaxTotal(20);//max connection
+
+        System.setProperty("jsse.enableSNIExtension", "false"); //""
+        CloseableHttpClient httpClient = builder
+                .setSSLSocketFactory(sslsf)
+                .setConnectionManager(cm)
+                .build();
+
+        client.setHttpclient(httpClient);
+        return client;
+    }
+    public static void main(String... argv) throws Exception {
+        URL endpoint = new URL("http://hailataxii.com/taxii-discovery-service");
+        String username = "guest";
+        String password = "guest";
+        TaxiiConnectionConfig config = new TaxiiConnectionConfig();
+        config = config.withConnectionType(ConnectionType.DISCOVER)
+                       .withEndpoint(endpoint)
+                       .withUsername(username)
+                       .withCollection("guest.Abuse_ch")
+                       .withPassword(password);
+        //TaxiiHandler handler = new TaxiiHandler(config, null);
+        //handler.run();
+        //discoverPollingClient(null, endpoint, username, password, context);
+    }
+}


[3/6] incubator-metron git commit: METRON-50: Ingest threat intel data from Taxii feeds closes apache/incubator-metron#29

Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/test/resources/taxii-messages/messages.poll
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/test/resources/taxii-messages/messages.poll b/metron-streaming/Metron-DataLoads/src/test/resources/taxii-messages/messages.poll
new file mode 100644
index 0000000..1c9d529
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/test/resources/taxii-messages/messages.poll
@@ -0,0 +1,2914 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<taxii_11:Poll_Response collection_name="guest.Abuse_ch" more="false" in_response_to="urn:uuid:8bb2bae7-cc8a-43ae-ab81-f581e6e97a7e" message_id="36900" xmlns:xmldsig="http://www.w3.org/2000/09/xmldsig#" xmlns:taxii_11="http://taxii.mitre.org/messages/taxii_xml_binding-1.1">
+    <taxii_11:Inclusive_End_Timestamp>2016-02-22T15:24:02.950562Z</taxii_11:Inclusive_End_Timestamp>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-16853623-bf9e-4691-a602-c9e0a9b4777a" timestamp="2016-02-22T15:24:02.958672+00:00" version="1.1.1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:indicator="http://stix.mitre.org/Indicator-2" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:ttp="http://stix.mitre.org/TTP-1" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCom
 mon="http://cybox.mitre.org/common-2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Indicators>
+                    <stix:Indicator id="opensource:indicator-e04fe4b7-82ae-4586-99bc-40b8c1d99304" timestamp="2014-10-31T16:44:24.973043+00:00" version="2.1.1" xsi:type="indicator:IndicatorType">
+                        <indicator:Title>ZeuS Tracker (offline)| www.office-112.com/wp-blog/offi.bin (2014-10-13) | This domain has been identified as malicious by zeustracker.abuse.ch</indicator:Title>
+                        <indicator:Type xsi:type="stixVocabs:IndicatorTypeVocab-1.1">Domain Watchlist</indicator:Type>
+                        <indicator:Type xsi:type="stixVocabs:IndicatorTypeVocab-1.1">URL Watchlist</indicator:Type>
+                        <indicator:Type xsi:type="stixVocabs:IndicatorTypeVocab-1.1">File Hash Watchlist</indicator:Type>
+                        <indicator:Description>This domain www.office-112.com has been identified as malicious by zeustracker.abuse.ch. For more detailed infomation about this indicator go to [CAUTION!!Read-URL-Before-Click] [https://zeustracker.abuse.ch/monitor.php?host=www.office-112.com].</indicator:Description>
+                        <indicator:Observable idref="opensource:Observable-c374c3ab-5c34-46bb-aa56-2e2d33de8c18">
+            </indicator:Observable>
+                        <indicator:Indicated_TTP>
+                            <stixCommon:TTP idref="opensource:ttp-48eaa91e-c331-4e1d-89e3-254b440cd927" xsi:type="ttp:TTPType"/>
+            </indicator:Indicated_TTP>
+                        <indicator:Producer>
+                            <stixCommon:Identity id="opensource:Identity-cae1a346-9f33-488f-af6e-1109692473ee">
+                                <stixCommon:Name>zeustracker.abuse.ch</stixCommon:Name>
+                </stixCommon:Identity>
+                            <stixCommon:Time>
+                                <cyboxCommon:Produced_Time>2014-10-13T00:00:00+00:00</cyboxCommon:Produced_Time>
+                                <cyboxCommon:Received_Time>2014-10-20T19:29:30+00:00</cyboxCommon:Received_Time>
+                </stixCommon:Time>
+            </indicator:Producer>
+        </stix:Indicator>
+    </stix:Indicators>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.960198Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-b1c62719-4936-4c1e-822f-43596dc85a98" timestamp="2016-02-22T15:24:02.960865+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:ttp="http://stix.mitre.org/TTP-1" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.org/common-2" xmlns:cybox="h
 ttp://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:TTPs>
+                    <stix:TTP id="opensource:ttp-48eaa91e-c331-4e1d-89e3-254b440cd927" timestamp="2014-10-31T16:44:24.974454+00:00" version="1.1.1" xsi:type="ttp:TTPType">
+                        <ttp:Title>ZeuS</ttp:Title>
+                        <ttp:Behavior>
+                            <ttp:Malware>
+                                <ttp:Malware_Instance id="opensource:malware-4eafc7e8-833c-4807-986c-bfbca8a9e86e">
+                                    <ttp:Type xsi:type="stixVocabs:MalwareTypeVocab-1.0">Remote Access Trojan</ttp:Type>
+                                    <ttp:Name>ZeuS</ttp:Name>
+                                    <ttp:Name>Zbot</ttp:Name>
+                                    <ttp:Name>Zeus</ttp:Name>
+                                    <ttp:Description>Zeus, ZeuS, or Zbot is Trojan horse computer malware that runs on computers running under versions of the Microsoft Windows operating system. While it is capable of being used to carry out many malicious and criminal tasks, it is often used to steal banking information by man-in-the-browser keystroke logging and form grabbing. It is also used to install the CryptoLocker ransomware.[1] Zeus is spread mainly through drive-by downloads and phishing schemes. (2014(http://en.wikipedia.org/wiki/Zeus_%28Trojan_horse%29))</ttp:Description>
+                                    <ttp:Short_Description>Zeus, ZeuS, or Zbot is Trojan horse computer malware effects Microsoft Windows operating system</ttp:Short_Description>
+                    </ttp:Malware_Instance>
+                </ttp:Malware>
+            </ttp:Behavior>
+        </stix:TTP>
+    </stix:TTPs>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.961541Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-466aa3c2-3c73-4e17-856a-299c0ce8e53b" timestamp="2016-02-22T15:24:02.962045+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.org/common-2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi
 ="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+                    <cybox:Observable id="opensource:Observable-c374c3ab-5c34-46bb-aa56-2e2d33de8c18">
+                        <cybox:Observable_Composition operator="OR">
+                            <cybox:Observable idref="opensource:Observable-b2084701-ab03-4233-b3ee-4cf96e3b9131">
+                </cybox:Observable>
+                            <cybox:Observable idref="opensource:Observable-84495ed8-634c-4ef8-aeb0-5968e1ef065b">
+                </cybox:Observable>
+                            <cybox:Observable idref="opensource:Observable-61b0fce7-df20-4f5f-a410-a2e1f80debc4">
+                </cybox:Observable>
+            </cybox:Observable_Composition>
+        </cybox:Observable>
+    </stix:Observables>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.962573Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-4aac2ef7-e5ae-413b-9da8-80457a3a5a1c" timestamp="2016-02-22T15:24:02.963049+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:FileObj="http://cybox.mitre.org/objects#FileObject-2" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.org/comm
 on-2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+                    <cybox:Observable id="opensource:Observable-61b0fce7-df20-4f5f-a410-a2e1f80debc4" sighting_count="1">
+                        <cybox:Title>File: offi.bin</cybox:Title>
+                        <cybox:Description>FileName: offi.bin | FileHash: f4004af2ad5e52fc9a67c5950978b141 | </cybox:Description>
+                        <cybox:Object id="opensource:File-15a9628d-42c7-43c8-ac4c-fc1237db2cdb">
+                            <cybox:Properties xsi:type="FileObj:FileObjectType">
+                                <FileObj:File_Name>offi.bin</FileObj:File_Name>
+                                <FileObj:File_Format>bin</FileObj:File_Format>
+                                <FileObj:Hashes>
+                                    <cyboxCommon:Hash>
+                                        <cyboxCommon:Type xsi:type="cyboxVocabs:HashNameVocab-1.0">MD5</cyboxCommon:Type>
+                                        <cyboxCommon:Simple_Hash_Value condition="Equals">f4004af2ad5e52fc9a67c5950978b141</cyboxCommon:Simple_Hash_Value>
+                        </cyboxCommon:Hash>
+                    </FileObj:Hashes>
+                </cybox:Properties>
+            </cybox:Object>
+        </cybox:Observable>
+    </stix:Observables>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.964187Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-7724b0a5-e820-4320-b1aa-6bee89070ea9" timestamp="2016-02-22T15:24:02.964677+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.org/common-2" xmlns:URIObj="http://cybox.mitre.org/objects#URIObject
 -2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+                    <cybox:Observable id="opensource:Observable-84495ed8-634c-4ef8-aeb0-5968e1ef065b" sighting_count="1">
+                        <cybox:Title>URI: http://www.office-112.com/wp-blog/offi.bin</cybox:Title>
+                        <cybox:Description>URI: http://www.office-112.com/wp-blog/offi.bin | Type: URL | </cybox:Description>
+                        <cybox:Object id="opensource:URI-a56683cd-0553-46ee-b197-431928b184b1">
+                            <cybox:Properties type="URL" xsi:type="URIObj:URIObjectType">
+                                <URIObj:Value condition="Equals">http://www.office-112.com/wp-blog/offi.bin</URIObj:Value>
+                </cybox:Properties>
+            </cybox:Object>
+        </cybox:Observable>
+    </stix:Observables>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.965365Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-ee87f913-a31c-4fa8-84c9-d7b5212f3483" timestamp="2016-02-22T15:24:02.965950+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:DomainNameObj="http://cybox.mitre.org/objects#DomainNameObject-1" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mi
 tre.org/common-2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+                    <cybox:Observable id="opensource:Observable-b2084701-ab03-4233-b3ee-4cf96e3b9131" sighting_count="1">
+                        <cybox:Title>Domain: www.office-112.com</cybox:Title>
+                        <cybox:Description>Domain: www.office-112.com | isFQDN: True | </cybox:Description>
+                        <cybox:Object id="opensource:DomainName-045c58ac-5fb4-4b84-95a2-19cf91c1e166">
+                            <cybox:Properties xsi:type="DomainNameObj:DomainNameObjectType">
+                                <DomainNameObj:Value condition="Equals">www.office-112.com</DomainNameObj:Value>
+                </cybox:Properties>
+            </cybox:Object>
+        </cybox:Observable>
+    </stix:Observables>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.966541Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-faf794a7-b20f-4ef2-8b81-08d706799b9d" timestamp="2016-02-22T15:24:02.966875+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:indicator="http://stix.mitre.org/Indicator-2" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:ttp="http://stix.mitre.org/TTP-1" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCom
 mon="http://cybox.mitre.org/common-2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Indicators>
+                    <stix:Indicator id="opensource:indicator-5c797902-13ea-4b3f-abb4-867eab337185" timestamp="2014-10-31T16:44:24.723668+00:00" version="2.1.1" xsi:type="indicator:IndicatorType">
+                        <indicator:Title>ZeuS Tracker (offline)| 94.102.53.142/~zadmin/find/http.bin (2014-10-14) | This IP address has been identified as malicious by zeustracker.abuse.ch</indicator:Title>
+                        <indicator:Type xsi:type="stixVocabs:IndicatorTypeVocab-1.1">IP Watchlist</indicator:Type>
+                        <indicator:Type xsi:type="stixVocabs:IndicatorTypeVocab-1.1">URL Watchlist</indicator:Type>
+                        <indicator:Type xsi:type="stixVocabs:IndicatorTypeVocab-1.1">File Hash Watchlist</indicator:Type>
+                        <indicator:Description>This IP address 94.102.53.142 has been identified as malicious by zeustracker.abuse.ch. For more detailed infomation about this indicator go to [CAUTION!!Read-URL-Before-Click] [https://zeustracker.abuse.ch/monitor.php?host=94.102.53.142].</indicator:Description>
+                        <indicator:Observable idref="opensource:Observable-87fd80af-647f-43ab-8992-3fe478593793">
+            </indicator:Observable>
+                        <indicator:Indicated_TTP>
+                            <stixCommon:TTP idref="opensource:ttp-08b96668-60fe-4a85-b28e-31fc9fe917c2" xsi:type="ttp:TTPType"/>
+            </indicator:Indicated_TTP>
+                        <indicator:Producer>
+                            <stixCommon:Identity id="opensource:Identity-6c77d05c-9929-4d50-bcc7-edd06125ca38">
+                                <stixCommon:Name>zeustracker.abuse.ch</stixCommon:Name>
+                </stixCommon:Identity>
+                            <stixCommon:Time>
+                                <cyboxCommon:Produced_Time>2014-10-14T00:00:00+00:00</cyboxCommon:Produced_Time>
+                                <cyboxCommon:Received_Time>2014-10-20T19:29:30+00:00</cyboxCommon:Received_Time>
+                </stixCommon:Time>
+            </indicator:Producer>
+        </stix:Indicator>
+    </stix:Indicators>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.968141Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-61200a02-db0c-42dd-a156-0d8c230ffcde" timestamp="2016-02-22T15:24:02.968669+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.org/common-2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi
 ="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+                    <cybox:Observable id="opensource:Observable-87fd80af-647f-43ab-8992-3fe478593793">
+                        <cybox:Observable_Composition operator="OR">
+                            <cybox:Observable idref="opensource:Observable-2aa6d9ec-633a-4dbf-9410-64efce2422ff">
+                </cybox:Observable>
+                            <cybox:Observable idref="opensource:Observable-010363d5-cca4-47ac-8538-23a1151fbcfd">
+                </cybox:Observable>
+                            <cybox:Observable idref="opensource:Observable-e49fe4fb-6aef-486d-be21-0ec8a371ddb1">
+                </cybox:Observable>
+            </cybox:Observable_Composition>
+        </cybox:Observable>
+    </stix:Observables>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.969150Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-c56b73b0-9e74-4f98-adfe-4d3fbe49fd66" timestamp="2016-02-22T15:24:02.969617+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.org/common-2" xmlns:URIObj="http://cybox.mitre.org/objects#URIObject
 -2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+                    <cybox:Observable id="opensource:Observable-010363d5-cca4-47ac-8538-23a1151fbcfd" sighting_count="1">
+                        <cybox:Title>URI: http://94.102.53.142/~zadmin/find/http.bin</cybox:Title>
+                        <cybox:Description>URI: http://94.102.53.142/~zadmin/find/http.bin | Type: URL | </cybox:Description>
+                        <cybox:Object id="opensource:URI-38f2db8f-41d6-4876-850b-5283e1987270">
+                            <cybox:Properties type="URL" xsi:type="URIObj:URIObjectType">
+                                <URIObj:Value condition="Equals">http://94.102.53.142/~zadmin/find/http.bin</URIObj:Value>
+                </cybox:Properties>
+            </cybox:Object>
+        </cybox:Observable>
+    </stix:Observables>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.970273Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-aba08109-b0aa-4852-9e95-05463debcdc0" timestamp="2016-02-22T15:24:02.970755+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:AddressObj="http://cybox.mitre.org/objects#AddressObject-2" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.or
 g/common-2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+                    <cybox:Observable id="opensource:Observable-2aa6d9ec-633a-4dbf-9410-64efce2422ff" sighting_count="1">
+                        <cybox:Title>IP: 94.102.53.142</cybox:Title>
+                        <cybox:Description>IPv4: 94.102.53.142 | isSource: True | </cybox:Description>
+                        <cybox:Object id="opensource:Address-cf5489ab-a097-46df-9ec9-91ab3bbcc477">
+                            <cybox:Properties category="ipv4-addr" is_source="true" xsi:type="AddressObj:AddressObjectType">
+                                <AddressObj:Address_Value condition="Equal">94.102.53.142</AddressObj:Address_Value>
+                </cybox:Properties>
+            </cybox:Object>
+        </cybox:Observable>
+    </stix:Observables>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.971398Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-77c7b736-eb95-4055-8bae-61c5e2bc4310" timestamp="2016-02-22T15:24:02.971847+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:FileObj="http://cybox.mitre.org/objects#FileObject-2" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.org/comm
 on-2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+                    <cybox:Observable id="opensource:Observable-e49fe4fb-6aef-486d-be21-0ec8a371ddb1" sighting_count="1">
+                        <cybox:Title>File: http.bin</cybox:Title>
+                        <cybox:Description>FileName: http.bin | FileHash: c88a8635a7eed7ff6641868b697650db | </cybox:Description>
+                        <cybox:Object id="opensource:File-546da1e6-6c00-4c0b-90f3-d56d4aa7f855">
+                            <cybox:Properties xsi:type="FileObj:FileObjectType">
+                                <FileObj:File_Name>http.bin</FileObj:File_Name>
+                                <FileObj:File_Format>bin</FileObj:File_Format>
+                                <FileObj:Hashes>
+                                    <cyboxCommon:Hash>
+                                        <cyboxCommon:Type xsi:type="cyboxVocabs:HashNameVocab-1.0">MD5</cyboxCommon:Type>
+                                        <cyboxCommon:Simple_Hash_Value condition="Equals">c88a8635a7eed7ff6641868b697650db</cyboxCommon:Simple_Hash_Value>
+                        </cyboxCommon:Hash>
+                    </FileObj:Hashes>
+                </cybox:Properties>
+            </cybox:Object>
+        </cybox:Observable>
+    </stix:Observables>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.972941Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-c5daa87d-9dbe-4cbe-b18b-2d01589c1f55" timestamp="2016-02-22T15:24:02.973486+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:ttp="http://stix.mitre.org/TTP-1" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.org/common-2" xmlns:cybox="h
 ttp://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:TTPs>
+                    <stix:TTP id="opensource:ttp-08b96668-60fe-4a85-b28e-31fc9fe917c2" timestamp="2014-10-31T16:44:24.724411+00:00" version="1.1.1" xsi:type="ttp:TTPType">
+                        <ttp:Title>ZeuS</ttp:Title>
+                        <ttp:Behavior>
+                            <ttp:Malware>
+                                <ttp:Malware_Instance id="opensource:malware-c3b59946-8e31-4f50-bbd9-132d418ecb7a">
+                                    <ttp:Type xsi:type="stixVocabs:MalwareTypeVocab-1.0">Remote Access Trojan</ttp:Type>
+                                    <ttp:Name>ZeuS</ttp:Name>
+                                    <ttp:Name>Zbot</ttp:Name>
+                                    <ttp:Name>Zeus</ttp:Name>
+                                    <ttp:Description>Zeus, ZeuS, or Zbot is Trojan horse computer malware that runs on computers running under versions of the Microsoft Windows operating system. While it is capable of being used to carry out many malicious and criminal tasks, it is often used to steal banking information by man-in-the-browser keystroke logging and form grabbing. It is also used to install the CryptoLocker ransomware.[1] Zeus is spread mainly through drive-by downloads and phishing schemes. (2014(http://en.wikipedia.org/wiki/Zeus_%28Trojan_horse%29))</ttp:Description>
+                                    <ttp:Short_Description>Zeus, ZeuS, or Zbot is Trojan horse computer malware effects Microsoft Windows operating system</ttp:Short_Description>
+                    </ttp:Malware_Instance>
+                </ttp:Malware>
+            </ttp:Behavior>
+        </stix:TTP>
+    </stix:TTPs>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.974363Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-926c768a-4e31-43cd-9390-37229e3fa2f7" timestamp="2016-02-22T15:24:02.974720+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:indicator="http://stix.mitre.org/Indicator-2" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:ttp="http://stix.mitre.org/TTP-1" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCom
 mon="http://cybox.mitre.org/common-2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Indicators>
+                    <stix:Indicator id="opensource:indicator-f93507d6-dad1-4299-8617-dff154b5ac62" timestamp="2014-10-31T16:44:24.911510+00:00" version="2.1.1" xsi:type="indicator:IndicatorType">
+                        <indicator:Title>ZeuS Tracker (online)| krlsma.com/wp-includes/Text/dom/php/file.php (2014-10-15) | This domain has been identified as malicious by zeustracker.abuse.ch</indicator:Title>
+                        <indicator:Type xsi:type="stixVocabs:IndicatorTypeVocab-1.1">Domain Watchlist</indicator:Type>
+                        <indicator:Type xsi:type="stixVocabs:IndicatorTypeVocab-1.1">URL Watchlist</indicator:Type>
+                        <indicator:Type xsi:type="stixVocabs:IndicatorTypeVocab-1.1">File Hash Watchlist</indicator:Type>
+                        <indicator:Description>This domain krlsma.com has been identified as malicious by zeustracker.abuse.ch. For more detailed infomation about this indicator go to [CAUTION!!Read-URL-Before-Click] [https://zeustracker.abuse.ch/monitor.php?host=krlsma.com].</indicator:Description>
+                        <indicator:Observable idref="opensource:Observable-24efd7b7-3474-4b22-8ad7-285f298dad41">
+            </indicator:Observable>
+                        <indicator:Indicated_TTP>
+                            <stixCommon:TTP idref="opensource:ttp-8f601fc3-bd6d-4c4e-94c9-b5dbad93ed0b" xsi:type="ttp:TTPType"/>
+            </indicator:Indicated_TTP>
+                        <indicator:Producer>
+                            <stixCommon:Identity id="opensource:Identity-740330e4-9f30-4710-9336-ff9d71492984">
+                                <stixCommon:Name>zeustracker.abuse.ch</stixCommon:Name>
+                </stixCommon:Identity>
+                            <stixCommon:Time>
+                                <cyboxCommon:Produced_Time>2014-10-15T00:00:00+00:00</cyboxCommon:Produced_Time>
+                                <cyboxCommon:Received_Time>2014-10-20T19:29:30+00:00</cyboxCommon:Received_Time>
+                </stixCommon:Time>
+            </indicator:Producer>
+        </stix:Indicator>
+    </stix:Indicators>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.975911Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-6225344f-12c1-49f8-b6e2-862d804044b1" timestamp="2016-02-22T15:24:02.976445+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:ttp="http://stix.mitre.org/TTP-1" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.org/common-2" xmlns:cybox="h
 ttp://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:TTPs>
+                    <stix:TTP id="opensource:ttp-8f601fc3-bd6d-4c4e-94c9-b5dbad93ed0b" timestamp="2014-10-31T16:44:24.912948+00:00" version="1.1.1" xsi:type="ttp:TTPType">
+                        <ttp:Title>ZeuS</ttp:Title>
+                        <ttp:Behavior>
+                            <ttp:Malware>
+                                <ttp:Malware_Instance id="opensource:malware-a908f65f-1a21-4658-b70c-ea1acd6ceca1">
+                                    <ttp:Type xsi:type="stixVocabs:MalwareTypeVocab-1.0">Remote Access Trojan</ttp:Type>
+                                    <ttp:Name>ZeuS</ttp:Name>
+                                    <ttp:Name>Zbot</ttp:Name>
+                                    <ttp:Name>Zeus</ttp:Name>
+                                    <ttp:Description>Zeus, ZeuS, or Zbot is Trojan horse computer malware that runs on computers running under versions of the Microsoft Windows operating system. While it is capable of being used to carry out many malicious and criminal tasks, it is often used to steal banking information by man-in-the-browser keystroke logging and form grabbing. It is also used to install the CryptoLocker ransomware.[1] Zeus is spread mainly through drive-by downloads and phishing schemes. (2014(http://en.wikipedia.org/wiki/Zeus_%28Trojan_horse%29))</ttp:Description>
+                                    <ttp:Short_Description>Zeus, ZeuS, or Zbot is Trojan horse computer malware effects Microsoft Windows operating system</ttp:Short_Description>
+                    </ttp:Malware_Instance>
+                </ttp:Malware>
+            </ttp:Behavior>
+        </stix:TTP>
+    </stix:TTPs>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.977154Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-3d34d4a5-9c0f-4cbb-9751-ced4d77dfc89" timestamp="2016-02-22T15:24:02.977643+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.org/common-2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi
 ="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+                    <cybox:Observable id="opensource:Observable-24efd7b7-3474-4b22-8ad7-285f298dad41">
+                        <cybox:Observable_Composition operator="OR">
+                            <cybox:Observable idref="opensource:Observable-cf7403a3-6c63-49e1-ab97-ed0861f71ba9">
+                </cybox:Observable>
+                            <cybox:Observable idref="opensource:Observable-5dade958-25b7-4eec-b3de-c25b580d6d6c">
+                </cybox:Observable>
+                            <cybox:Observable idref="opensource:Observable-398c91f0-d12b-499b-844c-67c7a0a9dbe1">
+                </cybox:Observable>
+            </cybox:Observable_Composition>
+        </cybox:Observable>
+    </stix:Observables>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.978111Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-1a737ebf-ee76-4460-9cb2-183577366ab9" timestamp="2016-02-22T15:24:02.978580+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:DomainNameObj="http://cybox.mitre.org/objects#DomainNameObject-1" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mi
 tre.org/common-2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+                    <cybox:Observable id="opensource:Observable-cf7403a3-6c63-49e1-ab97-ed0861f71ba9" sighting_count="1">
+                        <cybox:Title>Domain: krlsma.com</cybox:Title>
+                        <cybox:Description>Domain: krlsma.com | isFQDN: True | </cybox:Description>
+                        <cybox:Object id="opensource:DomainName-53c73da2-81ef-4242-98b2-1b1215e0f124">
+                            <cybox:Properties xsi:type="DomainNameObj:DomainNameObjectType">
+                                <DomainNameObj:Value condition="Equals">krlsma.com</DomainNameObj:Value>
+                </cybox:Properties>
+            </cybox:Object>
+        </cybox:Observable>
+    </stix:Observables>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.979274Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-8f2994a9-1a61-40fc-be8f-63f7709823d1" timestamp="2016-02-22T15:24:02.979721+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.org/common-2" xmlns:URIObj="http://cybox.mitre.org/objects#URIObject
 -2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+                    <cybox:Observable id="opensource:Observable-5dade958-25b7-4eec-b3de-c25b580d6d6c" sighting_count="1">
+                        <cybox:Title>URI: http://krlsma.com/wp-includes/Text/dom/php/file.php</cybox:Title>
+                        <cybox:Description>URI: http://krlsma.com/wp-includes/Text/dom/php/file.php | Type: URL | </cybox:Description>
+                        <cybox:Object id="opensource:URI-44fa3e9f-6599-4dda-bbe6-a03b06930084">
+                            <cybox:Properties type="URL" xsi:type="URIObj:URIObjectType">
+                                <URIObj:Value condition="Equals">http://krlsma.com/wp-includes/Text/dom/php/file.php</URIObj:Value>
+                </cybox:Properties>
+            </cybox:Object>
+        </cybox:Observable>
+    </stix:Observables>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.980513Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-935dbc5c-5eb5-4022-ba49-95d30b845160" timestamp="2016-02-22T15:24:02.981129+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:FileObj="http://cybox.mitre.org/objects#FileObject-2" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.org/comm
 on-2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+                    <cybox:Observable id="opensource:Observable-398c91f0-d12b-499b-844c-67c7a0a9dbe1" sighting_count="1">
+                        <cybox:Title>File: file.php</cybox:Title>
+                        <cybox:Description>FileName: file.php | FileHash: cccc3d971cc7f2814229e836076664a1 | </cybox:Description>
+                        <cybox:Object id="opensource:File-eb908acc-cf77-4cd4-9875-eb6408c4c726">
+                            <cybox:Properties xsi:type="FileObj:FileObjectType">
+                                <FileObj:File_Name>file.php</FileObj:File_Name>
+                                <FileObj:File_Format>php</FileObj:File_Format>
+                                <FileObj:Hashes>
+                                    <cyboxCommon:Hash>
+                                        <cyboxCommon:Type xsi:type="cyboxVocabs:HashNameVocab-1.0">MD5</cyboxCommon:Type>
+                                        <cyboxCommon:Simple_Hash_Value condition="Equals">cccc3d971cc7f2814229e836076664a1</cyboxCommon:Simple_Hash_Value>
+                        </cyboxCommon:Hash>
+                    </FileObj:Hashes>
+                </cybox:Properties>
+            </cybox:Object>
+        </cybox:Observable>
+    </stix:Observables>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.982272Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-28ffd222-1203-4609-86c3-efa5cfac9b41" timestamp="2016-02-22T15:24:02.982590+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:indicator="http://stix.mitre.org/Indicator-2" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:ttp="http://stix.mitre.org/TTP-1" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCom
 mon="http://cybox.mitre.org/common-2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Indicators>
+                    <stix:Indicator id="opensource:indicator-0324e19d-9a5a-4cc0-bc74-5a04b6de8bd3" timestamp="2014-10-31T16:44:24.842915+00:00" version="2.1.1" xsi:type="indicator:IndicatorType">
+                        <indicator:Title>ZeuS Tracker (offline)| goomjav1kaformjavkd.com/neverwind/tmp/pixel.jpg (2014-10-31) | This domain has been identified as malicious by zeustracker.abuse.ch</indicator:Title>
+                        <indicator:Type xsi:type="stixVocabs:IndicatorTypeVocab-1.1">Domain Watchlist</indicator:Type>
+                        <indicator:Type xsi:type="stixVocabs:IndicatorTypeVocab-1.1">URL Watchlist</indicator:Type>
+                        <indicator:Type xsi:type="stixVocabs:IndicatorTypeVocab-1.1">File Hash Watchlist</indicator:Type>
+                        <indicator:Description>This domain goomjav1kaformjavkd.com has been identified as malicious by zeustracker.abuse.ch. For more detailed infomation about this indicator go to [CAUTION!!Read-URL-Before-Click] [https://zeustracker.abuse.ch/monitor.php?host=goomjav1kaformjavkd.com].</indicator:Description>
+                        <indicator:Observable idref="opensource:Observable-eae86c81-73ec-4ee6-87b6-31a8fa3fe5ac">
+            </indicator:Observable>
+                        <indicator:Indicated_TTP>
+                            <stixCommon:TTP idref="opensource:ttp-e6a4b409-9e89-4841-b293-f08483efb12f" xsi:type="ttp:TTPType"/>
+            </indicator:Indicated_TTP>
+                        <indicator:Producer>
+                            <stixCommon:Identity id="opensource:Identity-3b2e3f22-c0ae-4f57-aac0-f7da7de9d294">
+                                <stixCommon:Name>zeustracker.abuse.ch</stixCommon:Name>
+                </stixCommon:Identity>
+                            <stixCommon:Time>
+                                <cyboxCommon:Produced_Time>2014-10-31T00:00:00+00:00</cyboxCommon:Produced_Time>
+                                <cyboxCommon:Received_Time>2014-10-31T16:44:24+00:00</cyboxCommon:Received_Time>
+                </stixCommon:Time>
+            </indicator:Producer>
+        </stix:Indicator>
+    </stix:Indicators>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.983991Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-8c7a08a2-8c9d-4761-aebc-0b7a0aac5f68" timestamp="2016-02-22T15:24:02.984608+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:ttp="http://stix.mitre.org/TTP-1" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.org/common-2" xmlns:cybox="h
 ttp://cybox.mitre.org/cybox-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:TTPs>
+                    <stix:TTP id="opensource:ttp-e6a4b409-9e89-4841-b293-f08483efb12f" timestamp="2014-10-31T16:44:24.843868+00:00" version="1.1.1" xsi:type="ttp:TTPType">
+                        <ttp:Title>ZeuS</ttp:Title>
+                        <ttp:Behavior>
+                            <ttp:Malware>
+                                <ttp:Malware_Instance id="opensource:malware-d0ef452d-4e25-4770-b560-4e9c01e0de25">
+                                    <ttp:Type xsi:type="stixVocabs:MalwareTypeVocab-1.0">Remote Access Trojan</ttp:Type>
+                                    <ttp:Name>ZeuS</ttp:Name>
+                                    <ttp:Name>Zbot</ttp:Name>
+                                    <ttp:Name>Zeus</ttp:Name>
+                                    <ttp:Description>Zeus, ZeuS, or Zbot is Trojan horse computer malware that runs on computers running under versions of the Microsoft Windows operating system. While it is capable of being used to carry out many malicious and criminal tasks, it is often used to steal banking information by man-in-the-browser keystroke logging and form grabbing. It is also used to install the CryptoLocker ransomware.[1] Zeus is spread mainly through drive-by downloads and phishing schemes. (2014(http://en.wikipedia.org/wiki/Zeus_%28Trojan_horse%29))</ttp:Description>
+                                    <ttp:Short_Description>Zeus, ZeuS, or Zbot is Trojan horse computer malware effects Microsoft Windows operating system</ttp:Short_Description>
+                    </ttp:Malware_Instance>
+                </ttp:Malware>
+            </ttp:Behavior>
+        </stix:TTP>
+    </stix:TTPs>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.985825Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-6042ee98-aa0a-47c0-982c-97fc6c8b65b8" timestamp="2016-02-22T15:24:02.986544+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2" xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1" xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1" xmlns:stix="http://stix.mitre.org/stix-1" xmlns:marking="http://data-marking.mitre.org/Marking-1" xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1" xmlns:cyboxCommon="http://cybox.mitre.org/common-2" xmlns:cybox="http://cybox.mitre.org/cybox-2" xmlns:xsi
 ="http://www.w3.org/2001/XMLSchema-instance">
+                <stix:STIX_Header>
+                    <stix:Handling>
+                        <marking:Marking>
+                            <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure>
+                            <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/>
+                            <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType">
+                                <TOUMarking:Terms_Of_Use>zeustracker.abuse.ch | Abuse source[https://sslbl.abuse.ch/blacklist/] - As for all abuse.ch projects, the use of the SSL Blacklist is free for both commercial and non-commercial usage without any limitation. However, if you are a commercial vendor of security software/services and you want to integrate data from the SSL Blacklist into your products / services, you will have to ask for permission first by contacting me using the contact form [http://www.abuse.ch/?page_id=4727].'
+</TOUMarking:Terms_Of_Use>
+                </marking:Marking_Structure>
+                            <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType">
+                                <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement>
+                </marking:Marking_Structure>
+            </marking:Marking>
+        </stix:Handling>
+    </stix:STIX_Header>
+                <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0">
+                    <cybox:Observable id="opensource:Observable-eae86c81-73ec-4ee6-87b6-31a8fa3fe5ac">
+                        <cybox:Observable_Composition operator="OR">
+                            <cybox:Observable idref="opensource:Observable-013f5351-5e03-4256-8405-ab3342146755">
+                </cybox:Observable>
+                            <cybox:Observable idref="opensource:Observable-3bb01333-456b-4f2c-9d40-d35f08702f74">
+                </cybox:Observable>
+                            <cybox:Observable idref="opensource:Observable-c0974652-6074-4410-aa49-3cf828a39663">
+                </cybox:Observable>
+            </cybox:Observable_Composition>
+        </cybox:Observable>
+    </stix:Observables>
+</stix:STIX_Package>
+        </taxii_11:Content>
+        <taxii_11:Timestamp_Label>2016-02-22T15:24:02.987068Z</taxii_11:Timestamp_Label>
+    </taxii_11:Content_Block>
+    <taxii_11:Content_Block>
+        <taxii_11:Content_Binding binding_id="urn:stix.mitre.org:xml:1.1.1"/>
+        <taxii_11:Content>
+            <stix:STIX_Package id="edge:Package-f74faad4-5796-4f11-955c-fc8d23852fe1" timestamp="2016-02-22T15:24:02.987620+00:00" version="1.1.1" xmlns:stixCommon="http://stix.mitre.org/common-1" xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1" xmlns:edge="http://soltra.com/" xmlns:DomainNameObj="http://cybox.mitre.org/objects#DomainNameObject-1" xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1" xmlns:opensource="http://hailataxii.com" xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularie

<TRUNCATED>


[4/6] incubator-metron git commit: METRON-50: Ingest threat intel data from Taxii feeds closes apache/incubator-metron#29

Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TaxiiLoader.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TaxiiLoader.java b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TaxiiLoader.java
new file mode 100644
index 0000000..4f307c5
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/main/java/org/apache/metron/dataloads/taxii/TaxiiLoader.java
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.dataloads.taxii;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import org.apache.commons.cli.*;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.log4j.PropertyConfigurator;
+import org.apache.metron.dataloads.extractor.Extractor;
+import org.apache.metron.dataloads.extractor.ExtractorHandler;
+import org.apache.metron.dataloads.extractor.stix.StixExtractor;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.text.*;
+import java.util.Date;
+import java.util.Map;
+import java.util.Timer;
+
+public class TaxiiLoader {
+private static abstract class OptionHandler implements Function<String, Option> {}
+    private enum TaxiiOptions {
+        HELP("h", new OptionHandler() {
+
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                return new Option(s, "help", false, "Generate Help screen");
+            }
+        })
+        ,EXTRACTOR_CONFIG("e", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "extractor_config", true, "JSON Document describing the extractor for this input data source");
+                o.setArgName("JSON_FILE");
+                o.setRequired(true);
+                return o;
+            }
+        })
+        ,CONNECTION_CONFIG("c", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "taxii_connection_config", true, "The JSON config file to configure the connection");
+                o.setArgName("config_file");
+                o.setRequired(true);
+                return o;
+            }
+        })
+        ,TIME_BETWEEN_POLLS("p", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "time_between_polls", true, "The time between polls (in ms)");
+                o.setArgName("MS");
+                o.setRequired(false);
+                return o;
+            }
+        })
+        ,BEGIN_TIME("b", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "begin_time", true, "Start time to poll the Taxii server (all data from that point will be gathered in the first pull).");
+                o.setArgName(DATE_FORMAT.toPattern());
+                o.setRequired(false);
+                return o;
+            }
+        })
+        ,LOG4J_PROPERTIES("l", new OptionHandler() {
+            @Nullable
+            @Override
+            public Option apply(@Nullable String s) {
+                Option o = new Option(s, "log4j", true, "The log4j properties file to load");
+                o.setArgName("FILE");
+                o.setRequired(false);
+                return o;
+            }
+        })
+        ;
+        Option option;
+        String shortCode;
+        TaxiiOptions(String shortCode, OptionHandler optionHandler) {
+            this.shortCode = shortCode;
+            this.option = optionHandler.apply(shortCode);
+        }
+
+        public boolean has(CommandLine cli) {
+            return cli.hasOption(shortCode);
+        }
+
+        public String get(CommandLine cli) {
+            return cli.getOptionValue(shortCode);
+        }
+
+        public static CommandLine parse(CommandLineParser parser, String[] args) {
+            try {
+                CommandLine cli = parser.parse(getOptions(), args);
+                if(TaxiiOptions.HELP.has(cli)) {
+                    printHelp();
+                    System.exit(0);
+                }
+                return cli;
+            } catch (ParseException e) {
+                System.err.println("Unable to parse args: " + Joiner.on(' ').join(args));
+                e.printStackTrace(System.err);
+                printHelp();
+                System.exit(-1);
+                return null;
+            }
+        }
+
+        public static void printHelp() {
+            HelpFormatter formatter = new HelpFormatter();
+            formatter.printHelp( "TaxiiLoader", getOptions());
+        }
+
+        public static Options getOptions() {
+            Options ret = new Options();
+            for(TaxiiOptions o : TaxiiOptions.values()) {
+               ret.addOption(o.option);
+            }
+            return ret;
+        }
+    }
+    public static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+    public static final long ONE_HR_IN_MS = 60*60*1000;
+    public static final long DEFAULT_TIME_BETWEEN_POLLS = ONE_HR_IN_MS;
+    public static void main(String... argv) throws Exception {
+        Configuration conf = HBaseConfiguration.create();
+        String[] otherArgs = new GenericOptionsParser(conf, argv).getRemainingArgs();
+
+        CommandLine cli = TaxiiOptions.parse(new PosixParser(), otherArgs);
+        if(TaxiiOptions.LOG4J_PROPERTIES.has(cli)) {
+            PropertyConfigurator.configure(TaxiiOptions.LOG4J_PROPERTIES.get(cli));
+        }
+        ExtractorHandler handler = ExtractorHandler.load(FileUtils.readFileToString(new File(TaxiiOptions.EXTRACTOR_CONFIG.get(cli))));
+        Extractor e = handler.getExtractor();
+        Timer timer = new Timer();
+        if(e instanceof StixExtractor) {
+            StixExtractor extractor = (StixExtractor)e;
+            TaxiiConnectionConfig connectionConfig = TaxiiConnectionConfig.load(FileUtils.readFileToString(new File(TaxiiOptions.CONNECTION_CONFIG.get(cli))));
+            if(TaxiiOptions.BEGIN_TIME.has(cli)) {
+                Date d = DATE_FORMAT.parse(TaxiiOptions.BEGIN_TIME.get(cli));
+                connectionConfig.withBeginTime(d);
+            }
+            long timeBetween = DEFAULT_TIME_BETWEEN_POLLS;
+            if(TaxiiOptions.TIME_BETWEEN_POLLS.has(cli)) {
+                timeBetween = Long.parseLong(TaxiiOptions.TIME_BETWEEN_POLLS.get(cli));
+            }
+            timer.scheduleAtFixedRate(new TaxiiHandler(connectionConfig, extractor, conf), 0, timeBetween);
+        }
+        else {
+            throw new IllegalStateException("Extractor must be a STIX Extractor");
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/ExtractorTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/ExtractorTest.java b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/ExtractorTest.java
index 519b95a..a124710 100644
--- a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/ExtractorTest.java
+++ b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/ExtractorTest.java
@@ -18,7 +18,9 @@
 package org.apache.metron.dataloads.extractor;
 
 import com.google.common.collect.Iterables;
-import org.apache.metron.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
+import org.apache.metron.reference.lookup.LookupKV;
 import org.apache.metron.threatintel.ThreatIntelResults;
 import org.junit.Assert;
 import org.junit.Test;
@@ -33,12 +35,12 @@ public class ExtractorTest {
     {
 
         @Override
-        public Iterable<ThreatIntelResults> extract(String line) throws IOException {
+        public Iterable<LookupKV> extract(String line) throws IOException {
             ThreatIntelKey key = new ThreatIntelKey();
             key.indicator = "dummy";
             Map<String, String> value = new HashMap<>();
             value.put("indicator", "dummy");
-            return Arrays.asList(new ThreatIntelResults(key, value));
+            return Arrays.asList(new LookupKV(key, new ThreatIntelValue(value)));
         }
 
         @Override
@@ -49,9 +51,11 @@ public class ExtractorTest {
     @Test
     public void testDummyExtractor() throws IllegalAccessException, InstantiationException, ClassNotFoundException, IOException {
         Extractor extractor = Extractors.create(DummyExtractor.class.getName());
-        ThreatIntelResults results = Iterables.getFirst(extractor.extract(null), null);
-        Assert.assertEquals("dummy", results.getKey().indicator);
-        Assert.assertEquals("dummy", results.getValue().get("indicator"));
+        LookupKV results = Iterables.getFirst(extractor.extract(null), null);
+        ThreatIntelKey key = (ThreatIntelKey) results.getKey();
+        ThreatIntelValue value = (ThreatIntelValue) results.getValue();
+        Assert.assertEquals("dummy", key.indicator);
+        Assert.assertEquals("dummy", value.getMetadata().get("indicator"));
     }
 
     @Test
@@ -68,8 +72,10 @@ public class ExtractorTest {
                 "            ,\"extractor\" : \"org.apache.metron.dataloads.extractor.ExtractorTest$DummyExtractor\"\n" +
                 "         }";
         ExtractorHandler handler = ExtractorHandler.load(config);
-        ThreatIntelResults results = Iterables.getFirst(handler.getExtractor().extract(null), null);
-        Assert.assertEquals("dummy", results.getKey().indicator);
-        Assert.assertEquals("dummy", results.getValue().get("indicator"));
+        LookupKV results = Iterables.getFirst(handler.getExtractor().extract(null), null);
+        ThreatIntelKey key = (ThreatIntelKey) results.getKey();
+        ThreatIntelValue value = (ThreatIntelValue) results.getValue();
+        Assert.assertEquals("dummy", key.indicator);
+        Assert.assertEquals("dummy", value.getMetadata().get("indicator"));
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java
index 96d9046..682b230 100644
--- a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java
+++ b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/csv/CSVExtractorTest.java
@@ -19,6 +19,9 @@ package org.apache.metron.dataloads.extractor.csv;
 
 import com.google.common.collect.Iterables;
 import org.apache.metron.dataloads.extractor.ExtractorHandler;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
+import org.apache.metron.reference.lookup.LookupKV;
 import org.apache.metron.threatintel.ThreatIntelResults;
 import org.junit.Assert;
 import org.junit.Test;
@@ -102,14 +105,16 @@ public class CSVExtractorTest {
 
     public void validate(ExtractorHandler handler) throws IOException {
         {
-            ThreatIntelResults results = Iterables.getFirst(handler.getExtractor().extract("google.com,1.0,foo"), null);
-            Assert.assertEquals("google.com", results.getKey().indicator);
-            Assert.assertEquals("google.com", results.getValue().get("host"));
-            Assert.assertEquals("foo", results.getValue().get("meta"));
-            Assert.assertEquals(2, results.getValue().size());
+            LookupKV results = Iterables.getFirst(handler.getExtractor().extract("google.com,1.0,foo"), null);
+            ThreatIntelKey key = (ThreatIntelKey) results.getKey();
+            ThreatIntelValue value = (ThreatIntelValue) results.getValue();
+            Assert.assertEquals("google.com", key.indicator);
+            Assert.assertEquals("google.com", value.getMetadata().get("host"));
+            Assert.assertEquals("foo", value.getMetadata().get("meta"));
+            Assert.assertEquals(2, value.getMetadata().size());
         }
         {
-            Iterable<ThreatIntelResults> results = handler.getExtractor().extract("#google.com,1.0,foo");
+            Iterable<LookupKV> results = handler.getExtractor().extract("#google.com,1.0,foo");
             Assert.assertEquals(0, Iterables.size(results));
         }
     }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/stix/StixExtractorTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/stix/StixExtractorTest.java b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/stix/StixExtractorTest.java
index f25a5a0..eaf6677 100644
--- a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/stix/StixExtractorTest.java
+++ b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/extractor/stix/StixExtractorTest.java
@@ -18,15 +18,13 @@
 package org.apache.metron.dataloads.extractor.stix;
 
 import com.google.common.collect.Iterables;
-import org.apache.metron.dataloads.ThreatIntelBulkLoader;
 import org.apache.metron.dataloads.extractor.Extractor;
 import org.apache.metron.dataloads.extractor.ExtractorHandler;
-import org.apache.metron.threatintel.ThreatIntelResults;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+import org.apache.metron.reference.lookup.LookupKV;
 import org.junit.Assert;
 import org.junit.Test;
 
-import java.util.List;
-
 public class StixExtractorTest {
     @Test
     public void testStixAddresses() throws Exception {
@@ -148,11 +146,12 @@ public class StixExtractorTest {
                     "         }";
             ExtractorHandler handler = ExtractorHandler.load(config);
             Extractor extractor = handler.getExtractor();
-            Iterable<ThreatIntelResults> results = extractor.extract(stixDoc);
+            Iterable<LookupKV> results = extractor.extract(stixDoc);
+
             Assert.assertEquals(3, Iterables.size(results));
-            Assert.assertEquals("10.0.0.0", Iterables.get(results, 0).getKey().indicator);
-            Assert.assertEquals("10.0.0.1", Iterables.get(results, 1).getKey().indicator);
-            Assert.assertEquals("10.0.0.2", Iterables.get(results, 2).getKey().indicator);
+            Assert.assertEquals("10.0.0.0", ((ThreatIntelKey)(Iterables.get(results, 0).getKey())).indicator);
+            Assert.assertEquals("10.0.0.1", ((ThreatIntelKey)(Iterables.get(results, 1).getKey())).indicator);
+            Assert.assertEquals("10.0.0.2", ((ThreatIntelKey)(Iterables.get(results, 2).getKey())).indicator);
         }
         {
             /**
@@ -169,11 +168,11 @@ public class StixExtractorTest {
                     "         }";
             ExtractorHandler handler = ExtractorHandler.load(config);
             Extractor extractor = handler.getExtractor();
-            Iterable<ThreatIntelResults> results = extractor.extract(stixDoc);
+            Iterable<LookupKV> results = extractor.extract(stixDoc);
             Assert.assertEquals(3, Iterables.size(results));
-            Assert.assertEquals("10.0.0.0", Iterables.get(results, 0).getKey().indicator);
-            Assert.assertEquals("10.0.0.1", Iterables.get(results, 1).getKey().indicator);
-            Assert.assertEquals("10.0.0.2", Iterables.get(results, 2).getKey().indicator);
+            Assert.assertEquals("10.0.0.0", ((ThreatIntelKey)(Iterables.get(results, 0).getKey())).indicator);
+            Assert.assertEquals("10.0.0.1", ((ThreatIntelKey)(Iterables.get(results, 1).getKey())).indicator);
+            Assert.assertEquals("10.0.0.2", ((ThreatIntelKey)(Iterables.get(results, 2).getKey())).indicator);
         }
         {
             /**
@@ -192,7 +191,7 @@ public class StixExtractorTest {
                     "         }";
             ExtractorHandler handler = ExtractorHandler.load(config);
             Extractor extractor = handler.getExtractor();
-            Iterable<ThreatIntelResults> results = extractor.extract(stixDoc);
+            Iterable<LookupKV> results = extractor.extract(stixDoc);
             Assert.assertEquals(0, Iterables.size(results));
         }
     }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/HBaseConverterTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/HBaseConverterTest.java b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/HBaseConverterTest.java
deleted file mode 100644
index 3587e59..0000000
--- a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/HBaseConverterTest.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.dataloads.hbase;
-
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.metron.threatintel.hbase.Converter;
-import org.apache.metron.threatintel.ThreatIntelKey;
-import org.apache.metron.threatintel.ThreatIntelResults;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.AbstractMap;
-import java.util.HashMap;
-import java.util.Map;
-
-public class HBaseConverterTest {
-    ThreatIntelKey key = new ThreatIntelKey("google");
-    Map<String, String> value = new HashMap<String, String>() {{
-        put("foo", "bar");
-        put("grok", "baz");
-    }};
-    Long timestamp = 7L;
-    ThreatIntelResults results = new ThreatIntelResults(key, value);
-    @Test
-    public void testKeySerialization() {
-        byte[] serialized = key.toBytes();
-        ThreatIntelKey deserialized = ThreatIntelKey.fromBytes(serialized);
-        Assert.assertEquals(key, deserialized);
-    }
-
-    @Test
-    public void testPut() throws IOException {
-        Put put = Converter.INSTANCE.toPut("cf", key, value, timestamp);
-        Map.Entry<ThreatIntelResults, Long> converted= Converter.INSTANCE.fromPut(put, "cf");
-        Assert.assertEquals(new AbstractMap.SimpleEntry<>(results, timestamp), converted);
-    }
-    @Test
-    public void testResult() throws IOException {
-        Result r = Converter.INSTANCE.toResult("cf", key, value, timestamp);
-        Map.Entry<ThreatIntelResults, Long> converted= Converter.INSTANCE.fromResult(r, "cf");
-        Assert.assertEquals(new AbstractMap.SimpleEntry<>(results, timestamp), converted);
-    }
-
-    @Test
-    public void testGet() throws Exception {
-        Get get = Converter.INSTANCE.toGet("cf", key);
-        Assert.assertArrayEquals(key.toBytes(), get.getRow());
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/HBaseThreatIntelConverterTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/HBaseThreatIntelConverterTest.java b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/HBaseThreatIntelConverterTest.java
new file mode 100644
index 0000000..21a1d26
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/HBaseThreatIntelConverterTest.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.dataloads.hbase;
+
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.metron.hbase.converters.HbaseConverter;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelConverter;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
+import org.apache.metron.reference.lookup.LookupKV;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+/**
+ * Created by cstella on 2/3/16.
+ */
+public class HBaseThreatIntelConverterTest {
+    ThreatIntelKey key = new ThreatIntelKey("google");
+    ThreatIntelValue value = new ThreatIntelValue(
+    new HashMap<String, String>() {{
+        put("foo", "bar");
+        put("grok", "baz");
+    }});
+    LookupKV<ThreatIntelKey, ThreatIntelValue> results = new LookupKV(key, value);
+    @Test
+    public void testKeySerialization() {
+        byte[] serialized = key.toBytes();
+
+        ThreatIntelKey deserialized = new ThreatIntelKey();
+        deserialized.fromBytes(serialized);
+        Assert.assertEquals(key, deserialized);
+    }
+
+    @Test
+    public void testPut() throws IOException {
+        HbaseConverter<ThreatIntelKey, ThreatIntelValue> converter = new ThreatIntelConverter();
+        Put put = converter.toPut("cf", key, value);
+        LookupKV<ThreatIntelKey, ThreatIntelValue> converted= converter.fromPut(put, "cf");
+        Assert.assertEquals(results, converted);
+    }
+    @Test
+    public void testResult() throws IOException {
+        HbaseConverter<ThreatIntelKey, ThreatIntelValue> converter = new ThreatIntelConverter();
+        Result r = converter.toResult("cf", key, value);
+        LookupKV<ThreatIntelKey, ThreatIntelValue> converted= converter.fromResult(r, "cf");
+        Assert.assertEquals(results, converted);
+    }
+
+    @Test
+    public void testGet() throws Exception {
+        HbaseConverter<ThreatIntelKey, ThreatIntelValue> converter = new ThreatIntelConverter();
+        Get get = converter.toGet("cf", key);
+        Assert.assertArrayEquals(key.toBytes(), get.getRow());
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperIntegrationTest.java b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperIntegrationTest.java
index 2824de8..be9ac90 100644
--- a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperIntegrationTest.java
+++ b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperIntegrationTest.java
@@ -26,9 +26,11 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.mapreduce.Job;
-import org.apache.metron.dataloads.ThreatIntelBulkLoader;
-import org.apache.metron.threatintel.ThreatIntelResults;
-import org.apache.metron.threatintel.hbase.Converter;
+import org.apache.metron.dataloads.bulk.ThreatIntelBulkLoader;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
+import org.apache.metron.reference.lookup.LookupKV;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelConverter;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -87,19 +89,19 @@ public class BulkLoadMapperIntegrationTest {
     Assert.assertNotNull(testTable);
     FileSystem fs = FileSystem.get(config);
     String contents = "google.com,1,foo";
+    ThreatIntelConverter converter = new ThreatIntelConverter();
     HBaseUtil.INSTANCE.writeFile(contents, new Path("input.csv"), fs);
-    Job job = ThreatIntelBulkLoader.createJob(config, "input.csv", tableName, cf, extractorConfig, 0L);
+    Job job = ThreatIntelBulkLoader.createJob(config, "input.csv", tableName, cf, extractorConfig, 0L, new ThreatIntelConverter());
     Assert.assertTrue(job.waitForCompletion(true));
     ResultScanner scanner = testTable.getScanner(Bytes.toBytes(cf));
-    List<Map.Entry<ThreatIntelResults, Long>> results = new ArrayList<>();
+    List<LookupKV<ThreatIntelKey, ThreatIntelValue>> results = new ArrayList<>();
     for(Result r : scanner) {
-      results.add(Converter.INSTANCE.fromResult(r, cf));
+      results.add(converter.fromResult(r, cf));
     }
     Assert.assertEquals(1, results.size());
-    Assert.assertEquals(0L, (long)results.get(0).getValue());
-    Assert.assertEquals(results.get(0).getKey().getKey().indicator, "google.com");
-    Assert.assertEquals(results.get(0).getKey().getValue().size(), 2);
-    Assert.assertEquals(results.get(0).getKey().getValue().get("meta"), "foo");
-    Assert.assertEquals(results.get(0).getKey().getValue().get("host"), "google.com");
+    Assert.assertEquals(results.get(0).getKey().indicator, "google.com");
+    Assert.assertEquals(results.get(0).getValue().getMetadata().size(), 2);
+    Assert.assertEquals(results.get(0).getValue().getMetadata().get("meta"), "foo");
+    Assert.assertEquals(results.get(0).getValue().getMetadata().get("host"), "google.com");
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperTest.java b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperTest.java
index c98c256..8ace7bc 100644
--- a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperTest.java
+++ b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/BulkLoadMapperTest.java
@@ -21,9 +21,11 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.io.Text;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
+import org.apache.metron.reference.lookup.LookupKV;
 import org.apache.metron.threatintel.ThreatIntelResults;
-import org.apache.metron.threatintel.hbase.Converter;
-import org.apache.metron.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelConverter;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -67,6 +69,7 @@ public class BulkLoadMapperTest {
             set(BulkLoadMapper.COLUMN_FAMILY_KEY, "cf");
             set(BulkLoadMapper.CONFIG_KEY, extractorConfig);
             set(BulkLoadMapper.LAST_SEEN_KEY, "0");
+            set(BulkLoadMapper.CONVERTER_KEY, ThreatIntelConverter.class.getName());
         }});
         {
             mapper.map(null, new Text("#google.com,1,foo"), null);
@@ -78,14 +81,14 @@ public class BulkLoadMapperTest {
             ThreatIntelKey expectedKey = new ThreatIntelKey() {{
                 indicator = "google.com";
             }};
+            ThreatIntelConverter converter = new ThreatIntelConverter();
             Put put = puts.get(new ImmutableBytesWritable(expectedKey.toBytes()));
             Assert.assertNotNull(puts);
-            Map.Entry<ThreatIntelResults, Long> results = Converter.INSTANCE.fromPut(put, "cf");
-            Assert.assertEquals(0L, (long)results.getValue());
-            Assert.assertEquals(results.getKey().getKey().indicator, "google.com");
-            Assert.assertEquals(results.getKey().getValue().size(), 2);
-            Assert.assertEquals(results.getKey().getValue().get("meta"), "foo");
-            Assert.assertEquals(results.getKey().getValue().get("host"), "google.com");
+            LookupKV<ThreatIntelKey, ThreatIntelValue> results = converter.fromPut(put, "cf");
+            Assert.assertEquals(results.getKey().indicator, "google.com");
+            Assert.assertEquals(results.getValue().getMetadata().size(), 2);
+            Assert.assertEquals(results.getValue().getMetadata().get("meta"), "foo");
+            Assert.assertEquals(results.getValue().getMetadata().get("host"), "google.com");
         }
 
     }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/LeastRecentlyUsedPrunerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/LeastRecentlyUsedPrunerIntegrationTest.java b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/LeastRecentlyUsedPrunerIntegrationTest.java
index 6d5c71c..a7d28fb 100644
--- a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/LeastRecentlyUsedPrunerIntegrationTest.java
+++ b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/hbase/mr/LeastRecentlyUsedPrunerIntegrationTest.java
@@ -19,19 +19,17 @@ package org.apache.metron.dataloads.hbase.mr;
 
 import com.google.common.collect.Iterables;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.mapreduce.Job;
-import org.apache.metron.dataloads.LeastRecentlyUsedPruner;
+import org.apache.metron.dataloads.bulk.LeastRecentlyUsedPruner;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
 import org.apache.metron.reference.lookup.LookupKey;
-import org.apache.metron.reference.lookup.accesstracker.AccessTrackerUtil;
 import org.apache.metron.reference.lookup.accesstracker.BloomAccessTracker;
 import org.apache.metron.reference.lookup.accesstracker.PersistentAccessTracker;
-import org.apache.metron.threatintel.ThreatIntelKey;
-import org.apache.metron.threatintel.hbase.Converter;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelConverter;
 import org.apache.metron.threatintel.hbase.ThreatIntelLookup;
 import org.junit.After;
 import org.junit.Assert;
@@ -84,24 +82,26 @@ public class LeastRecentlyUsedPrunerIntegrationTest {
         List<LookupKey> goodKeysOtherHalf = getKeys(5, 10);
         Iterable<LookupKey> goodKeys = Iterables.concat(goodKeysHalf, goodKeysOtherHalf);
         List<LookupKey> badKey = getKeys(10, 11);
+        ThreatIntelConverter converter = new ThreatIntelConverter();
         for(LookupKey k : goodKeysHalf) {
-            testTable.put(Converter.INSTANCE.toPut(cf, (ThreatIntelKey) k
-                                                  , new HashMap<String, String>() {{
+            testTable.put(converter.toPut(cf, (ThreatIntelKey) k
+                                            , new ThreatIntelValue(
+                                                  new HashMap<String, String>() {{
                                                     put("k", "dummy");
                                                     }}
-                                                  , 1L
                                                   )
+                                          )
                          );
             Assert.assertTrue(lookup.exists((ThreatIntelKey)k, testTable, true));
         }
         pat.persist(true);
         for(LookupKey k : goodKeysOtherHalf) {
-            testTable.put(Converter.INSTANCE.toPut(cf, (ThreatIntelKey) k
-                                                  , new HashMap<String, String>() {{
+            testTable.put(converter.toPut(cf, (ThreatIntelKey) k
+                                            , new ThreatIntelValue(new HashMap<String, String>() {{
                                                     put("k", "dummy");
                                                     }}
-                                                  , 1L
-                                                  )
+                                                                  )
+                                         )
                          );
             Assert.assertTrue(lookup.exists((ThreatIntelKey)k, testTable, true));
         }
@@ -112,11 +112,11 @@ public class LeastRecentlyUsedPrunerIntegrationTest {
         }
         pat.persist(true);
         {
-            testTable.put(Converter.INSTANCE.toPut(cf, (ThreatIntelKey) badKey.get(0)
-                    , new HashMap<String, String>() {{
+            testTable.put(converter.toPut(cf, (ThreatIntelKey) badKey.get(0)
+                    , new ThreatIntelValue(new HashMap<String, String>() {{
                         put("k", "dummy");
                     }}
-                    , 1L
+                    )
                     )
             );
         }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/taxii/MockTaxiiService.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/taxii/MockTaxiiService.java b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/taxii/MockTaxiiService.java
new file mode 100644
index 0000000..64e4618
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/taxii/MockTaxiiService.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.dataloads.taxii;
+
+
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.integration.util.UnitTestHelper;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.ApplicationPath;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.*;
+import javax.ws.rs.ext.RuntimeDelegate;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+@Path("/")
+public class MockTaxiiService {
+    static String discoveryMsg;
+    static String pollMsg;
+    static {
+        try {
+            String baseDir = UnitTestHelper.findDir("taxii-messages");
+            discoveryMsg = FileUtils.readFileToString(new File(new File(baseDir), "message.discovery"));
+            pollMsg= FileUtils.readFileToString(new File(new File(baseDir), "messages.poll"));
+        } catch (IOException e) {
+            throw new RuntimeException("Unable to read discovery message", e);
+        }
+    }
+
+
+    @POST
+    @Path("/taxii-discovery-service")
+    public Response getDiscovery() {
+        return Response.ok(discoveryMsg, MediaType.APPLICATION_XML_TYPE).header("x-taxii-content-type", "urn:taxii.mitre.org:message:xml:1.1").build();
+    }
+    @POST
+    @Path("/taxii-data")
+    public Response getData() {
+        return Response.ok(pollMsg).type(MediaType.APPLICATION_XML_TYPE).header("x-taxii-content-type", "urn:taxii.mitre.org:message:xml:1.1").build();
+    }
+
+    @ApplicationPath("rs")
+    public static class ApplicationConfig extends Application{
+        private final Set<Class<?>> classes;
+        public ApplicationConfig() {
+            HashSet<Class<?>> c = new HashSet<>();
+            c.add(MockTaxiiService.class);
+            classes = Collections.unmodifiableSet(c);
+        }
+        @Override
+        public Set<Class<?>> getClasses() {
+            return classes;
+        }
+    }
+    private static HttpServer server;
+    public static void start(int port) throws IOException {
+        // Create an HTTP server listening at port 8282
+        URI uri = UriBuilder.fromUri("http://localhost/").port(port).build();
+        server = HttpServer.create(new InetSocketAddress(uri.getPort()), 0);
+        HttpHandler handler = RuntimeDelegate.getInstance().createEndpoint(new ApplicationConfig(), HttpHandler.class);
+        server.createContext(uri.getPath(), handler);
+        discoveryMsg = discoveryMsg.replaceAll("PORT", "" + uri.getPort());
+        server.start();
+    }
+
+    public static void shutdown() {
+        if(server != null) {
+            server.stop(0);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/taxii/TaxiiIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/taxii/TaxiiIntegrationTest.java b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/taxii/TaxiiIntegrationTest.java
new file mode 100644
index 0000000..f61d729
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/test/java/org/apache/metron/dataloads/taxii/TaxiiIntegrationTest.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.dataloads.taxii;
+
+import com.google.common.base.Splitter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.metron.dataloads.extractor.stix.StixExtractor;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelConverter;
+import org.apache.metron.integration.util.UnitTestHelper;
+import org.apache.metron.integration.util.mock.MockHTable;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+public class TaxiiIntegrationTest {
+
+    @Before
+    public void setup() throws IOException {
+        MockTaxiiService.start(8282);
+    }
+
+    @After
+    public void teardown() {
+        MockTaxiiService.shutdown();
+        MockHTable.Provider.clear();
+    }
+
+
+
+    @Test
+    public void testTaxii() throws Exception {
+        /**
+         {
+            "endpoint" : "http://localhost:8282/taxii-discovery-service"
+           ,"type" : "DISCOVER"
+           ,"collection" : "guest.Abuse_ch"
+           ,"tableMap" : {
+                    "DomainName:FQDN" : "malicious_domain:cf"
+                   ,"Address:IPV_4_ADDR" : "malicious_address:cf"
+                         }
+         }
+         */
+        String taxiiConnectionConfig = "{\n" +
+                "            \"endpoint\" : \"http://localhost:8282/taxii-discovery-service\"\n" +
+                "           ,\"type\" : \"DISCOVER\"\n" +
+                "           ,\"collection\" : \"guest.Abuse_ch\"\n" +
+                "           ,\"tableMap\" : {\n" +
+                "                    \"DomainName:FQDN\" : \"malicious_domain:cf\"\n" +
+                "                   ,\"Address:IPV_4_ADDR\" : \"malicious_address:cf\"\n" +
+                "                         }\n" +
+                "         }";
+        final MockHTable.Provider provider = new MockHTable.Provider();
+        final Configuration config = HBaseConfiguration.create();
+        TaxiiHandler handler = new TaxiiHandler(TaxiiConnectionConfig.load(taxiiConnectionConfig), new StixExtractor(), config ) {
+            @Override
+            protected synchronized HTableInterface createHTable(TableInfo tableInfo) throws IOException {
+                return provider.addToCache(tableInfo.getTableName(), tableInfo.getColumnFamily());
+            }
+        };
+        //UnitTestHelper.verboseLogging();
+        handler.run();
+        Set<String> maliciousDomains;
+        {
+            MockHTable table = (MockHTable) provider.getTable(config, "malicious_domain");
+            maliciousDomains = getIndicators(table.getPutLog(), "cf");
+        }
+        Assert.assertTrue(maliciousDomains.contains("www.office-112.com"));
+        Assert.assertEquals(numStringsMatch(MockTaxiiService.pollMsg, "DomainNameObj:Value condition=\"Equals\""), maliciousDomains.size());
+        Set<String> maliciousAddresses;
+        {
+            MockHTable table = (MockHTable) provider.getTable(config, "malicious_address");
+            maliciousAddresses= getIndicators(table.getPutLog(), "cf");
+        }
+        Assert.assertTrue(maliciousAddresses.contains("94.102.53.142"));
+        Assert.assertEquals(numStringsMatch(MockTaxiiService.pollMsg, "AddressObj:Address_Value condition=\"Equal\""), maliciousAddresses.size());
+        MockHTable.Provider.clear();
+    }
+
+    private static int numStringsMatch(String xmlBundle, String text) {
+        int cnt = 0;
+        for(String line : Splitter.on("\n").split(xmlBundle)) {
+            if(line.contains(text)) {
+                cnt++;
+            }
+        }
+        return cnt;
+    }
+
+    private static Set<String> getIndicators(Iterable<Put> puts, String cf) throws IOException {
+        ThreatIntelConverter converter = new ThreatIntelConverter();
+        Set<String> ret = new HashSet<>();
+        for(Put p : puts) {
+            ret.add(converter.fromPut(p, cf).getKey().indicator);
+        }
+        return ret;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-DataLoads/src/test/resources/taxii-messages/message.discovery
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/src/test/resources/taxii-messages/message.discovery b/metron-streaming/Metron-DataLoads/src/test/resources/taxii-messages/message.discovery
new file mode 100644
index 0000000..2f7e788
--- /dev/null
+++ b/metron-streaming/Metron-DataLoads/src/test/resources/taxii-messages/message.discovery
@@ -0,0 +1,21 @@
+<taxii_11:Discovery_Response in_response_to="urn:uuid:695149bd-72ab-41dd-a7e0-7d0b2a73e81f" message_id="65257" xmlns:xmldsig="http://www.w3.org/2000/09/xmldsig#" xmlns:taxii_11="http://taxii.mitre.org/messages/taxii_xml_binding-1.1">
+    <taxii_11:Service_Instance service_type="DISCOVERY" available="true" service_version="urn:taxii.mitre.org:services:1.1">
+        <taxii_11:Protocol_Binding>urn:taxii.mitre.org:protocol:https:1.0</taxii_11:Protocol_Binding>
+        <taxii_11:Address>http://localhost:PORT/taxii-data</taxii_11:Address>
+        <taxii_11:Message_Binding>urn:taxii.mitre.org:message:xml:1.1</taxii_11:Message_Binding>
+        <taxii_11:Message></taxii_11:Message>
+    </taxii_11:Service_Instance>
+    <taxii_11:Service_Instance service_type="COLLECTION_MANAGEMENT" available="true" service_version="urn:taxii.mitre.org:services:1.1">
+        <taxii_11:Protocol_Binding>urn:taxii.mitre.org:protocol:https:1.0</taxii_11:Protocol_Binding>
+        <taxii_11:Address>http://localhost:PORT/taxii-collections</taxii_11:Address>
+        <taxii_11:Message_Binding>urn:taxii.mitre.org:message:xml:1.1</taxii_11:Message_Binding>
+        <taxii_11:Message></taxii_11:Message>
+    </taxii_11:Service_Instance>
+    <taxii_11:Service_Instance service_type="POLL" available="true" service_version="urn:taxii.mitre.org:services:1.1">
+        <taxii_11:Protocol_Binding>urn:taxii.mitre.org:protocol:https:1.0</taxii_11:Protocol_Binding>
+        <taxii_11:Address>http://localhost:PORT/taxii-data</taxii_11:Address>
+        <taxii_11:Message_Binding>urn:taxii.mitre.org:message:xml:1.1</taxii_11:Message_Binding>
+        <taxii_11:Message></taxii_11:Message>
+    </taxii_11:Service_Instance>
+</taxii_11:Discovery_Response>
+


[2/6] incubator-metron git commit: METRON-50: Ingest threat intel data from Taxii feeds closes apache/incubator-metron#29

Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/threatintel/ThreatIntelAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/threatintel/ThreatIntelAdapter.java b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/threatintel/ThreatIntelAdapter.java
index 8a82828..9db6398 100644
--- a/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/threatintel/ThreatIntelAdapter.java
+++ b/metron-streaming/Metron-EnrichmentAdapters/src/main/java/org/apache/metron/threatintel/ThreatIntelAdapter.java
@@ -19,10 +19,8 @@ package org.apache.metron.threatintel;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-import org.apache.metron.reference.lookup.Lookup;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
 import org.apache.metron.reference.lookup.accesstracker.BloomAccessTracker;
 import org.apache.metron.reference.lookup.accesstracker.PersistentAccessTracker;
 import org.apache.metron.threatintel.hbase.ThreatIntelLookup;
@@ -32,7 +30,6 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.Serializable;
-import java.util.Map;
 import java.util.UUID;
 
 public class ThreatIntelAdapter implements EnrichmentAdapter<String>,Serializable {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Indexing/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Indexing/pom.xml b/metron-streaming/Metron-Indexing/pom.xml
index 2c80a8d..e2a1037 100644
--- a/metron-streaming/Metron-Indexing/pom.xml
+++ b/metron-streaming/Metron-Indexing/pom.xml
@@ -24,7 +24,6 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-        <elastic.search.version>1.3.1</elastic.search.version>
         <http.client.version>4.3.4</http.client.version>
         <jsonsimple.version>1.1.1</jsonsimple.version>
     </properties>
@@ -74,7 +73,7 @@
         <dependency>
             <groupId>org.elasticsearch</groupId>
             <artifactId>elasticsearch</artifactId>
-            <version>${elastic.search.version}</version>
+            <version>${global_elasticsearch_version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
@@ -93,11 +92,6 @@
             <version>1.9</version>
             <scope>provided</scope>
         </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>3.8.2</version>
-        </dependency>
 
     </dependencies>
     <reporting>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Testing/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/pom.xml b/metron-streaming/Metron-Testing/pom.xml
new file mode 100644
index 0000000..d68d81d
--- /dev/null
+++ b/metron-streaming/Metron-Testing/pom.xml
@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 
+  Licensed to the Apache Software 
+  Foundation (ASF) under one or more contributor license agreements. See the 
+  NOTICE file distributed with this work for additional information regarding 
+  copyright ownership. The ASF licenses this file to You under the Apache License, 
+  Version 2.0 (the "License"); you may not use this file except in compliance 
+  with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+  Unless required by applicable law or agreed to in writing, software distributed 
+  under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES 
+  OR CONDITIONS OF ANY KIND, either express or implied. See the License for 
+  the specific language governing permissions and limitations under the License. 
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.metron</groupId>
+    <artifactId>Metron-Streaming</artifactId>
+    <version>0.1BETA</version>
+  </parent>
+  <artifactId>Metron-Testing</artifactId>
+  <description>Metron Testing Utilities</description>
+  <properties>
+  </properties>
+  <dependencies>
+    <dependency>
+      <groupId>log4j</groupId>
+      <artifactId>log4j</artifactId>
+      <version>1.2.17</version>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>${global_junit_version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.elasticsearch</groupId>
+      <artifactId>elasticsearch</artifactId>
+      <version>${global_elasticsearch_version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.storm</groupId>
+      <artifactId>flux-core</artifactId>
+      <version>${global_flux_version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.storm</groupId>
+      <artifactId>storm-core</artifactId>
+      <version>${global_storm_version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>${global_hbase_guava_version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-client</artifactId>
+      <version>${global_hbase_version}</version>
+      <exclusions>
+        <exclusion>
+          <artifactId>log4j</artifactId>
+          <groupId>log4j</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+      <version>${global_hadoop_version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+  </dependencies>
+
+  <build>
+  </build>
+  <reporting>
+  </reporting>
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/UnitTestHelper.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/UnitTestHelper.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/UnitTestHelper.java
new file mode 100644
index 0000000..7af2212
--- /dev/null
+++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/UnitTestHelper.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration.util;
+
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+import org.junit.Assert;
+
+import java.io.File;
+import java.util.Set;
+import java.util.Stack;
+
+public class UnitTestHelper {
+    public static String findDir(String name) {
+        return findDir(new File("."), name);
+    }
+
+    public static String findDir(File startDir, String name) {
+        Stack<File> s = new Stack<File>();
+        s.push(startDir);
+        while(!s.empty()) {
+            File parent = s.pop();
+            if(parent.getName().equalsIgnoreCase(name)) {
+                return parent.getAbsolutePath();
+            }
+            else {
+                File[] children = parent.listFiles();
+                if(children != null) {
+                    for (File child : children) {
+                        s.push(child);
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
+    public static <T> void assertSetEqual(String type, Set<T> expectedPcapIds, Set<T> found) {
+        boolean mismatch = false;
+        for(T f : found) {
+            if(!expectedPcapIds.contains(f)) {
+                mismatch = true;
+                System.out.println("Found " + type + " that I did not expect: " + f);
+            }
+        }
+        for(T expectedId : expectedPcapIds) {
+            if(!found.contains(expectedId)) {
+                mismatch = true;
+                System.out.println("Expected " + type + " that I did not index: " + expectedId);
+            }
+        }
+        Assert.assertFalse(mismatch);
+    }
+
+    public static void verboseLogging() {
+        verboseLogging("%d [%p|%c|%C{1}] %m%n", Level.ALL);
+    }
+    public static void verboseLogging(String pattern, Level level) {
+        ConsoleAppender console = new ConsoleAppender(); //create appender
+        //configure the appender
+        console.setLayout(new PatternLayout(pattern));
+        console.setThreshold(level);
+        console.activateOptions();
+        //add appender to any Logger (here is root)
+        Logger.getRootLogger().addAppender(console);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/ComponentRunner.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/ComponentRunner.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/ComponentRunner.java
new file mode 100644
index 0000000..3e5e793
--- /dev/null
+++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/ComponentRunner.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration.util.integration;
+
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class ComponentRunner {
+    public static class Builder {
+        LinkedHashMap<String, InMemoryComponent> components;
+        String[] startupOrder;
+        String[] shutdownOrder;
+        public Builder() {
+            components = new LinkedHashMap<String, InMemoryComponent>();
+        }
+
+        public Builder withComponent(String name, InMemoryComponent component) {
+            components.put(name, component);
+            return this;
+        }
+
+        public Builder withCustomStartupOrder(String[] startupOrder) {
+            this.startupOrder = startupOrder;
+            return this;
+        }
+        public Builder withCustomShutdownOrder(String[] shutdownOrder) {
+            this.shutdownOrder = shutdownOrder;
+            return this;
+        }
+        private static String[] toOrderedList(Map<String, InMemoryComponent> components) {
+            String[] ret = new String[components.size()];
+            int i = 0;
+            for(String component : components.keySet()) {
+                ret[i++] = component;
+            }
+            return ret;
+        }
+        public ComponentRunner build() {
+            if(shutdownOrder == null) {
+                shutdownOrder = toOrderedList(components);
+            }
+            if(startupOrder == null) {
+                startupOrder = toOrderedList(components);
+            }
+            return new ComponentRunner(components, startupOrder, shutdownOrder);
+        }
+
+    }
+
+    LinkedHashMap<String, InMemoryComponent> components;
+    String[] startupOrder;
+    String[] shutdownOrder;
+    public ComponentRunner( LinkedHashMap<String, InMemoryComponent> components
+                          , String[] startupOrder
+                          , String[] shutdownOrder
+                          )
+    {
+        this.components = components;
+        this.startupOrder = startupOrder;
+        this.shutdownOrder = shutdownOrder;
+
+    }
+
+    public <T extends InMemoryComponent> T getComponent(String name, Class<T> clazz) {
+        return clazz.cast(getComponents().get(name));
+    }
+
+    public LinkedHashMap<String, InMemoryComponent> getComponents() {
+        return components;
+    }
+
+    public void start() throws UnableToStartException {
+        for(String componentName : startupOrder) {
+            components.get(componentName).start();
+        }
+    }
+    public void stop() {
+        for(String componentName : shutdownOrder) {
+            components.get(componentName).stop();
+        }
+    }
+
+    public <T> T process(Processor<T> successState) {
+        return process(successState, 5, 30000, 120000);
+    }
+
+    public <T> T process(Processor<T> successState, int numRetries, long timeBetweenAttempts, long maxTimeMs) {
+        int retryCount = 0;
+        long start = System.currentTimeMillis();
+        while(true) {
+            long duration = System.currentTimeMillis() - start;
+            if(duration > maxTimeMs) {
+                throw new RuntimeException("Took too long to complete: " + duration + " > " + maxTimeMs);
+            }
+            ReadinessState state = successState.process(this);
+            if(state == ReadinessState.READY) {
+                return successState.getResult();
+            }
+            else if(state == ReadinessState.NOT_READY) {
+                retryCount++;
+                if(numRetries > 0 && retryCount > numRetries) {
+                    throw new RuntimeException("Too many retries: " + retryCount);
+                }
+            }
+            try {
+                Thread.sleep(timeBetweenAttempts);
+            } catch (InterruptedException e) {
+                throw new RuntimeException("Unable to sleep", e);
+            }
+        }
+    }
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/InMemoryComponent.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/InMemoryComponent.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/InMemoryComponent.java
new file mode 100644
index 0000000..b5224a2
--- /dev/null
+++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/InMemoryComponent.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration.util.integration;
+
+public interface InMemoryComponent {
+    public void start() throws UnableToStartException;
+    public void stop();
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/Processor.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/Processor.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/Processor.java
new file mode 100644
index 0000000..59f3ece
--- /dev/null
+++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/Processor.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration.util.integration;
+
+public interface Processor<T> {
+    ReadinessState process(ComponentRunner runner);
+    T getResult();
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/ReadinessState.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/ReadinessState.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/ReadinessState.java
new file mode 100644
index 0000000..59ce69a
--- /dev/null
+++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/ReadinessState.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration.util.integration;
+
+public enum ReadinessState {
+    READY, NOT_READY;
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/UnableToStartException.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/UnableToStartException.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/UnableToStartException.java
new file mode 100644
index 0000000..4e691f4
--- /dev/null
+++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/UnableToStartException.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration.util.integration;
+
+public class UnableToStartException extends Exception {
+    public UnableToStartException(String message) {
+        super(message);
+    }
+    public UnableToStartException(String message, Throwable t) {
+        super(message, t);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/ElasticSearchComponent.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/ElasticSearchComponent.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/ElasticSearchComponent.java
new file mode 100644
index 0000000..a7991c0
--- /dev/null
+++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/ElasticSearchComponent.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration.util.integration.components;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.metron.integration.util.integration.InMemoryComponent;
+import org.apache.metron.integration.util.integration.UnableToStartException;
+import org.elasticsearch.ElasticsearchTimeoutException;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeBuilder;
+import org.elasticsearch.search.SearchHit;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class ElasticSearchComponent implements InMemoryComponent {
+
+    public static class Builder{
+        private int httpPort;
+        private File indexDir;
+        private Map<String, String> extraElasticSearchSettings = null;
+        public Builder withHttpPort(int httpPort) {
+            this.httpPort = httpPort;
+            return this;
+        }
+        public Builder withIndexDir(File indexDir) {
+            this.indexDir = indexDir;
+            return this;
+        }
+        public Builder withExtraElasticSearchSettings(Map<String, String> extraElasticSearchSettings) {
+            this.extraElasticSearchSettings = extraElasticSearchSettings;
+            return this;
+        }
+        public ElasticSearchComponent build() {
+            return new ElasticSearchComponent(httpPort, indexDir, extraElasticSearchSettings);
+        }
+    }
+
+    private Client client;
+    private Node node;
+    private int httpPort;
+    private File indexDir;
+    private Map<String, String> extraElasticSearchSettings;
+
+    public ElasticSearchComponent(int httpPort, File indexDir) {
+        this(httpPort, indexDir, null);
+    }
+    public ElasticSearchComponent(int httpPort, File indexDir, Map<String, String> extraElasticSearchSettings) {
+        this.httpPort = httpPort;
+        this.indexDir = indexDir;
+        this.extraElasticSearchSettings = extraElasticSearchSettings;
+    }
+    public Client getClient() {
+        return client;
+    }
+
+    private void cleanDir(File dir) throws IOException {
+        if(dir.exists()) {
+            FileUtils.deleteDirectory(dir);
+        }
+        dir.mkdirs();
+    }
+    public void start() throws UnableToStartException {
+        File logDir= new File(indexDir, "/logs");
+        File dataDir= new File(indexDir, "/data");
+        try {
+            cleanDir(logDir);
+            cleanDir(dataDir);
+
+        } catch (IOException e) {
+            throw new UnableToStartException("Unable to clean log or data directories", e);
+        }
+        ImmutableSettings.Builder immutableSettings = ImmutableSettings.settingsBuilder()
+                .put("node.http.enabled", true)
+                .put("http.port", httpPort)
+                .put("cluster.name", "metron")
+                .put("path.logs",logDir.getAbsolutePath())
+                .put("path.data",dataDir.getAbsolutePath())
+                .put("gateway.type", "none")
+                .put("index.store.type", "memory")
+                .put("index.number_of_shards", 1)
+                .put("node.mode", "network")
+                .put("index.number_of_replicas", 1);
+        if(extraElasticSearchSettings != null) {
+            immutableSettings = immutableSettings.put(extraElasticSearchSettings);
+        }
+        Settings settings = immutableSettings.build();
+
+        node = NodeBuilder.nodeBuilder().settings(settings).node();
+        node.start();
+        settings = ImmutableSettings.settingsBuilder()
+					.put("cluster.name", "metron").build();
+		client = new TransportClient(settings)
+					.addTransportAddress(new InetSocketTransportAddress("localhost",
+							9300));
+
+        waitForCluster(client, ClusterHealthStatus.YELLOW, new TimeValue(60000));
+    }
+
+    public static void waitForCluster(ElasticsearchClient client, ClusterHealthStatus status, TimeValue timeout) throws UnableToStartException {
+        try {
+            ClusterHealthResponse healthResponse =
+                    (ClusterHealthResponse)client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().waitForStatus(status).timeout(timeout)).actionGet();
+            if (healthResponse != null && healthResponse.isTimedOut()) {
+                throw new UnableToStartException("cluster state is " + healthResponse.getStatus().name()
+                        + " and not " + status.name()
+                        + ", from here on, everything will fail!");
+            }
+        } catch (ElasticsearchTimeoutException e) {
+            throw new UnableToStartException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
+        }
+    }
+
+    public List<Map<String, Object>> getAllIndexedDocs(String index) throws IOException {
+       return getAllIndexedDocs(index, "message");
+    }
+    public List<Map<String, Object>> getAllIndexedDocs(String index, String subMessage) throws IOException {
+        getClient().admin().indices().refresh(new RefreshRequest());
+        SearchResponse response = getClient().prepareSearch(index)
+                .setTypes("pcap_doc")
+                .setSource("message")
+                .setFrom(0)
+                .setSize(1000)
+                .execute().actionGet();
+        List<Map<String, Object>> ret = new ArrayList<Map<String, Object>>();
+        for (SearchHit hit : response.getHits()) {
+            Object o = null;
+            if(subMessage == null) {
+                o = hit.getSource();
+            }
+            else {
+                o = hit.getSource().get(subMessage);
+            }
+            ret.add((Map<String, Object>)(o));
+        }
+        return ret;
+    }
+    public boolean hasIndex(String indexName) {
+        Set<String> indices = getClient().admin()
+                                    .indices()
+                                    .stats(new IndicesStatsRequest())
+                                    .actionGet()
+                                    .getIndices()
+                                    .keySet();
+        return indices.contains(indexName);
+
+    }
+
+    public void stop() {
+        node.stop();
+        node = null;
+        client = null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/FluxTopologyComponent.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/FluxTopologyComponent.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/FluxTopologyComponent.java
new file mode 100644
index 0000000..2cac4ee
--- /dev/null
+++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/integration/components/FluxTopologyComponent.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration.util.integration.components;
+
+import backtype.storm.Config;
+import backtype.storm.LocalCluster;
+import backtype.storm.generated.StormTopology;
+import org.apache.metron.integration.util.integration.InMemoryComponent;
+import org.apache.metron.integration.util.integration.UnableToStartException;
+import org.apache.storm.flux.FluxBuilder;
+import org.apache.storm.flux.model.ExecutionContext;
+import org.apache.storm.flux.model.TopologyDef;
+import org.apache.storm.flux.parser.FluxParser;
+import org.apache.thrift7.TException;
+import org.junit.Assert;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Properties;
+
+public class FluxTopologyComponent implements InMemoryComponent {
+    LocalCluster stormCluster;
+    String topologyName;
+    File topologyLocation;
+    Properties topologyProperties;
+
+    public static class Builder {
+        String topologyName;
+        File topologyLocation;
+        Properties topologyProperties;
+        public Builder withTopologyName(String name) {
+            this.topologyName = name;
+            return this;
+        }
+        public Builder withTopologyLocation(File location) {
+            this.topologyLocation = location;
+            return this;
+        }
+        public Builder withTopologyProperties(Properties properties) {
+            this.topologyProperties = properties;
+            return this;
+        }
+
+        public FluxTopologyComponent build() {
+            return new FluxTopologyComponent(topologyName, topologyLocation, topologyProperties);
+        }
+    }
+
+    public FluxTopologyComponent(String topologyName, File topologyLocation, Properties topologyProperties) {
+        this.topologyName = topologyName;
+        this.topologyLocation = topologyLocation;
+        this.topologyProperties = topologyProperties;
+    }
+
+    public LocalCluster getStormCluster() {
+        return stormCluster;
+    }
+
+    public String getTopologyName() {
+        return topologyName;
+    }
+
+    public File getTopologyLocation() {
+        return topologyLocation;
+    }
+
+    public Properties getTopologyProperties() {
+        return topologyProperties;
+    }
+
+    public void start() throws UnableToStartException{
+        try {
+            stormCluster = new LocalCluster();
+        } catch (Exception e) {
+            throw new UnableToStartException("Unable to start flux topology: " + getTopologyLocation(), e);
+        }
+    }
+
+    public void stop() {
+        stormCluster.shutdown();
+    }
+
+    public void submitTopology() throws NoSuchMethodException, IOException, InstantiationException, TException, IllegalAccessException, InvocationTargetException, ClassNotFoundException {
+        startTopology(getTopologyName(), getTopologyLocation(), getTopologyProperties());
+    }
+    private void startTopology(String topologyName, File topologyLoc, Properties properties) throws IOException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, TException {
+        TopologyDef topologyDef = loadYaml(topologyName, topologyLoc, properties);
+        Config conf = FluxBuilder.buildConfig(topologyDef);
+        ExecutionContext context = new ExecutionContext(topologyDef, conf);
+        StormTopology topology = FluxBuilder.buildTopology(context);
+        Assert.assertNotNull(topology);
+        topology.validate();
+        stormCluster.submitTopology(topologyName, conf, topology);
+    }
+
+    private static TopologyDef loadYaml(String topologyName, File yamlFile, Properties properties) throws IOException {
+        File tmpFile = File.createTempFile(topologyName, "props");
+        tmpFile.deleteOnExit();
+        FileWriter propWriter = null;
+        try {
+            propWriter = new FileWriter(tmpFile);
+            properties.store(propWriter, topologyName + " properties");
+        }
+        finally {
+            if(propWriter != null) {
+                propWriter.close();
+                return FluxParser.parseFile(yamlFile.getAbsolutePath(), false, true, tmpFile.getAbsolutePath(), false);
+            }
+
+            return null;
+        }
+    }
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/mock/MockHTable.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/mock/MockHTable.java b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/mock/MockHTable.java
new file mode 100644
index 0000000..76558d5
--- /dev/null
+++ b/metron-streaming/Metron-Testing/src/main/java/org/apache/metron/integration/util/mock/MockHTable.java
@@ -0,0 +1,672 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration.util.mock;
+
+
+import com.google.protobuf.Descriptors;
+import com.google.protobuf.Message;
+import com.google.protobuf.Service;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.client.coprocessor.Batch;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.*;
+
+/**
+ * MockHTable.
+ *
+ * This implementation is a selected excerpt from https://gist.github.com/agaoglu/613217
+ */
+public class MockHTable implements HTableInterface {
+
+    public static class Provider implements Serializable {
+        private static Map<String, HTableInterface> _cache = new HashMap<>();
+        public HTableInterface getTable(Configuration config, String tableName) throws IOException {
+            return _cache.get(tableName);
+        }
+        public static HTableInterface getFromCache(String tableName) {
+            return _cache.get(tableName);
+        }
+        public static HTableInterface addToCache(String tableName, String... columnFamilies) {
+            MockHTable ret =  new MockHTable(tableName, columnFamilies);
+            _cache.put(tableName, ret);
+            return ret;
+        }
+
+        public static void clear() {
+            _cache.clear();
+        }
+    }
+
+    private final String tableName;
+    private final List<String> columnFamilies = new ArrayList<>();
+    private HColumnDescriptor[] descriptors;
+
+    private NavigableMap<byte[], NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>>> data
+            = new TreeMap<>(Bytes.BYTES_COMPARATOR);
+
+    private static List<KeyValue> toKeyValue(byte[] row, NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowdata, int maxVersions) {
+        return toKeyValue(row, rowdata, 0, Long.MAX_VALUE, maxVersions);
+    }
+
+    private static List<KeyValue> toKeyValue(byte[] row, NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowdata, long timestampStart, long timestampEnd, int maxVersions) {
+        List<KeyValue> ret = new ArrayList<KeyValue>();
+        for (byte[] family : rowdata.keySet())
+            for (byte[] qualifier : rowdata.get(family).keySet()) {
+                int versionsAdded = 0;
+                for (Map.Entry<Long, byte[]> tsToVal : rowdata.get(family).get(qualifier).descendingMap().entrySet()) {
+                    if (versionsAdded++ == maxVersions)
+                        break;
+                    Long timestamp = tsToVal.getKey();
+                    if (timestamp < timestampStart)
+                        continue;
+                    if (timestamp > timestampEnd)
+                        continue;
+                    byte[] value = tsToVal.getValue();
+                    ret.add(new KeyValue(row, family, qualifier, timestamp, value));
+                }
+            }
+        return ret;
+    }
+    public MockHTable(String tableName) {
+        this.tableName = tableName;
+    }
+
+    public MockHTable(String tableName, String... columnFamilies) {
+        this.tableName = tableName;
+        for(String cf : columnFamilies) {
+            addColumnFamily(cf);
+        }
+    }
+
+    public void addColumnFamily(String columnFamily) {
+        this.columnFamilies.add(columnFamily);
+        descriptors = new HColumnDescriptor[columnFamilies.size()];
+        int i = 0;
+        for(String cf : columnFamilies) {
+            descriptors[i++] = new HColumnDescriptor(cf);
+        }
+    }
+
+
+    @Override
+    public byte[] getTableName() {
+        return Bytes.toBytes(tableName);
+    }
+
+    @Override
+    public TableName getName() {
+        return TableName.valueOf(tableName);
+    }
+
+    @Override
+    public Configuration getConfiguration() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public HTableDescriptor getTableDescriptor() throws IOException {
+        HTableDescriptor ret = new HTableDescriptor(tableName);
+        for(HColumnDescriptor c : descriptors) {
+            ret.addFamily(c);
+        }
+        return ret;
+    }
+
+    @Override
+    public boolean exists(Get get) throws IOException {
+        if(get.getFamilyMap() == null || get.getFamilyMap().size() == 0) {
+            return data.containsKey(get.getRow());
+        } else {
+            byte[] row = get.getRow();
+            if(!data.containsKey(row)) {
+                return false;
+            }
+            for(byte[] family : get.getFamilyMap().keySet()) {
+                if(!data.get(row).containsKey(family)) {
+                    return false;
+                } else {
+                    return true;
+                }
+            }
+            return true;
+        }
+    }
+
+    /**
+     * Test for the existence of columns in the table, as specified by the Gets.
+     * <p/>
+     * <p/>
+     * This will return an array of booleans. Each value will be true if the related Get matches
+     * one or more keys, false if not.
+     * <p/>
+     * <p/>
+     * This is a server-side call so it prevents any data from being transferred to
+     * the client.
+     *
+     * @param gets the Gets
+     * @return Array of boolean.  True if the specified Get matches one or more keys, false if not.
+     * @throws IOException e
+     */
+    @Override
+    public boolean[] existsAll(List<Get> gets) throws IOException {
+        boolean[] ret = new boolean[gets.size()];
+        int i = 0;
+        for(boolean b : exists(gets)) {
+           ret[i++] = b;
+        }
+        return ret;
+    }
+
+    @Override
+    public Boolean[] exists(List<Get> list) throws IOException {
+        Boolean[] ret = new Boolean[list.size()];
+        int i = 0;
+        for(Get g : list) {
+           ret[i++] = exists(g);
+        }
+        return ret;
+    }
+
+    @Override
+    public void batch(List<? extends Row> list, Object[] objects) throws IOException, InterruptedException {
+        throw new UnsupportedOperationException();
+
+    }
+
+    /**
+     * @param actions
+     * @deprecated
+     */
+    @Override
+    public Object[] batch(List<? extends Row> actions) throws IOException, InterruptedException {
+        List<Result> results = new ArrayList<Result>();
+        for (Row r : actions) {
+            if (r instanceof Delete) {
+                delete((Delete) r);
+                continue;
+            }
+            if (r instanceof Put) {
+                put((Put) r);
+                continue;
+            }
+            if (r instanceof Get) {
+                results.add(get((Get) r));
+            }
+        }
+        return results.toArray();
+    }
+
+    @Override
+    public <R> void batchCallback(List<? extends Row> list, Object[] objects, Batch.Callback<R> callback) throws IOException, InterruptedException {
+        throw new UnsupportedOperationException();
+
+    }
+
+    /**
+     * @param list
+     * @param callback
+     * @deprecated
+     */
+    @Override
+    public <R> Object[] batchCallback(List<? extends Row> list, Batch.Callback<R> callback) throws IOException, InterruptedException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Result get(Get get) throws IOException {
+        if (!data.containsKey(get.getRow()))
+            return new Result();
+        byte[] row = get.getRow();
+        List<KeyValue> kvs = new ArrayList<KeyValue>();
+        if (!get.hasFamilies()) {
+            kvs = toKeyValue(row, data.get(row), get.getMaxVersions());
+        } else {
+            for (byte[] family : get.getFamilyMap().keySet()){
+                if (data.get(row).get(family) == null)
+                    continue;
+                NavigableSet<byte[]> qualifiers = get.getFamilyMap().get(family);
+                if (qualifiers == null || qualifiers.isEmpty())
+                    qualifiers = data.get(row).get(family).navigableKeySet();
+                for (byte[] qualifier : qualifiers){
+                    if (qualifier == null)
+                        qualifier = "".getBytes();
+                    if (!data.get(row).containsKey(family) ||
+                            !data.get(row).get(family).containsKey(qualifier) ||
+                            data.get(row).get(family).get(qualifier).isEmpty())
+                        continue;
+                    Map.Entry<Long, byte[]> timestampAndValue = data.get(row).get(family).get(qualifier).lastEntry();
+                    kvs.add(new KeyValue(row,family, qualifier, timestampAndValue.getKey(), timestampAndValue.getValue()));
+                }
+            }
+        }
+        Filter filter = get.getFilter();
+        if (filter != null) {
+            filter.reset();
+            List<KeyValue> nkvs = new ArrayList<KeyValue>(kvs.size());
+            for (KeyValue kv : kvs) {
+                if (filter.filterAllRemaining()) {
+                    break;
+                }
+                if (filter.filterRowKey(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength())) {
+                    continue;
+                }
+                if (filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE) {
+                    nkvs.add(kv);
+                }
+                // ignoring next key hint which is a optimization to reduce file system IO
+            }
+            if (filter.hasFilterRow()) {
+                filter.filterRow();
+            }
+            kvs = nkvs;
+        }
+
+        return new Result(kvs);
+    }
+
+    @Override
+    public Result[] get(List<Get> list) throws IOException {
+        Result[] ret = new Result[list.size()];
+        int i = 0;
+        for(Get g : list) {
+            ret[i++] = get(g);
+        }
+        return ret;
+    }
+
+    /**
+     * @param bytes
+     * @param bytes1
+     * @deprecated
+     */
+    @Override
+    public Result getRowOrBefore(byte[] bytes, byte[] bytes1) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ResultScanner getScanner(Scan scan) throws IOException {
+        final List<Result> ret = new ArrayList<Result>();
+        byte[] st = scan.getStartRow();
+        byte[] sp = scan.getStopRow();
+        Filter filter = scan.getFilter();
+
+        for (byte[] row : data.keySet()){
+            // if row is equal to startRow emit it. When startRow (inclusive) and
+            // stopRow (exclusive) is the same, it should not be excluded which would
+            // happen w/o this control.
+            if (st != null && st.length > 0 &&
+                    Bytes.BYTES_COMPARATOR.compare(st, row) != 0) {
+                // if row is before startRow do not emit, pass to next row
+                if (st != null && st.length > 0 &&
+                        Bytes.BYTES_COMPARATOR.compare(st, row) > 0)
+                    continue;
+                // if row is equal to stopRow or after it do not emit, stop iteration
+                if (sp != null && sp.length > 0 &&
+                        Bytes.BYTES_COMPARATOR.compare(sp, row) <= 0)
+                    break;
+            }
+
+            List<KeyValue> kvs = null;
+            if (!scan.hasFamilies()) {
+                kvs = toKeyValue(row, data.get(row), scan.getTimeRange().getMin(), scan.getTimeRange().getMax(), scan.getMaxVersions());
+            } else {
+                kvs = new ArrayList<KeyValue>();
+                for (byte[] family : scan.getFamilyMap().keySet()){
+                    if (data.get(row).get(family) == null)
+                        continue;
+                    NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(family);
+                    if (qualifiers == null || qualifiers.isEmpty())
+                        qualifiers = data.get(row).get(family).navigableKeySet();
+                    for (byte[] qualifier : qualifiers){
+                        if (data.get(row).get(family).get(qualifier) == null)
+                            continue;
+                        for (Long timestamp : data.get(row).get(family).get(qualifier).descendingKeySet()){
+                            if (timestamp < scan.getTimeRange().getMin())
+                                continue;
+                            if (timestamp > scan.getTimeRange().getMax())
+                                continue;
+                            byte[] value = data.get(row).get(family).get(qualifier).get(timestamp);
+                            kvs.add(new KeyValue(row, family, qualifier, timestamp, value));
+                            if(kvs.size() == scan.getMaxVersions()) {
+                                break;
+                            }
+                        }
+                    }
+                }
+            }
+            if (filter != null) {
+                filter.reset();
+                List<KeyValue> nkvs = new ArrayList<KeyValue>(kvs.size());
+                for (KeyValue kv : kvs) {
+                    if (filter.filterAllRemaining()) {
+                        break;
+                    }
+                    if (filter.filterRowKey(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength())) {
+                        continue;
+                    }
+                    Filter.ReturnCode filterResult = filter.filterKeyValue(kv);
+                    if (filterResult == Filter.ReturnCode.INCLUDE) {
+                        nkvs.add(kv);
+                    } else if (filterResult == Filter.ReturnCode.NEXT_ROW) {
+                        break;
+                    }
+                    // ignoring next key hint which is a optimization to reduce file system IO
+                }
+                if (filter.hasFilterRow()) {
+                    filter.filterRow();
+                }
+                kvs = nkvs;
+            }
+            if (!kvs.isEmpty()) {
+                ret.add(new Result(kvs));
+            }
+        }
+
+        return new ResultScanner() {
+            private final Iterator<Result> iterator = ret.iterator();
+            public Iterator<Result> iterator() {
+                return iterator;
+            }
+            public Result[] next(int nbRows) throws IOException {
+                ArrayList<Result> resultSets = new ArrayList<Result>(nbRows);
+                for(int i = 0; i < nbRows; i++) {
+                    Result next = next();
+                    if (next != null) {
+                        resultSets.add(next);
+                    } else {
+                        break;
+                    }
+                }
+                return resultSets.toArray(new Result[resultSets.size()]);
+            }
+            public Result next() throws IOException {
+                try {
+                    return iterator().next();
+                } catch (NoSuchElementException e) {
+                    return null;
+                }
+            }
+            public void close() {}
+        };
+    }
+    @Override
+    public ResultScanner getScanner(byte[] family) throws IOException {
+        Scan scan = new Scan();
+        scan.addFamily(family);
+        return getScanner(scan);
+    }
+
+    @Override
+    public ResultScanner getScanner(byte[] family, byte[] qualifier)
+            throws IOException {
+        Scan scan = new Scan();
+        scan.addColumn(family, qualifier);
+        return getScanner(scan);
+    }
+
+    List<Put> putLog = new ArrayList<>();
+
+    public List<Put> getPutLog() {
+        return putLog;
+    }
+
+    @Override
+    public void put(Put put) throws IOException {
+        putLog.add(put);
+        byte[] row = put.getRow();
+        NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowData = forceFind(data, row, new TreeMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>>(Bytes.BYTES_COMPARATOR));
+        for (byte[] family : put.getFamilyMap().keySet()){
+            NavigableMap<byte[], NavigableMap<Long, byte[]>> familyData = forceFind(rowData, family, new TreeMap<byte[], NavigableMap<Long, byte[]>>(Bytes.BYTES_COMPARATOR));
+            for (KeyValue kv : put.getFamilyMap().get(family)){
+                kv.updateLatestStamp(Bytes.toBytes(System.currentTimeMillis()));
+                byte[] qualifier = kv.getQualifier();
+                NavigableMap<Long, byte[]> qualifierData = forceFind(familyData, qualifier, new TreeMap<Long, byte[]>());
+                qualifierData.put(kv.getTimestamp(), kv.getValue());
+            }
+        }
+    }
+
+    /**
+     * Helper method to find a key in a map. If key is not found, newObject is
+     * added to map and returned
+     *
+     * @param map
+     *          map to extract value from
+     * @param key
+     *          key to look for
+     * @param newObject
+     *          set key to this if not found
+     * @return found value or newObject if not found
+     */
+    private <K, V> V forceFind(NavigableMap<K, V> map, K key, V newObject){
+        V data = map.get(key);
+        if (data == null){
+            data = newObject;
+            map.put(key, data);
+        }
+        return data;
+    }
+
+    @Override
+    public void put(List<Put> puts) throws IOException {
+        for (Put put : puts)
+            put(put);
+    }
+
+    @Override
+    public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, byte[] bytes3, Put put) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Atomically checks if a row/family/qualifier value matches the expected
+     * value. If it does, it adds the put.  If the passed value is null, the check
+     * is for the lack of column (ie: non-existance)
+     *
+     * @param row       to check
+     * @param family    column family to check
+     * @param qualifier column qualifier to check
+     * @param compareOp comparison operator to use
+     * @param value     the expected value
+     * @param put       data to put if check succeeds
+     * @return true if the new put was executed, false otherwise
+     * @throws IOException e
+     */
+    @Override
+    public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, CompareFilter.CompareOp compareOp, byte[] value, Put put) throws IOException {
+        return false;
+    }
+
+    @Override
+    public void delete(Delete delete) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void delete(List<Delete> list) throws IOException {
+        throw new UnsupportedOperationException();
+
+    }
+
+    @Override
+    public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, byte[] bytes3, Delete delete) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Atomically checks if a row/family/qualifier value matches the expected
+     * value. If it does, it adds the delete.  If the passed value is null, the
+     * check is for the lack of column (ie: non-existance)
+     *
+     * @param row       to check
+     * @param family    column family to check
+     * @param qualifier column qualifier to check
+     * @param compareOp comparison operator to use
+     * @param value     the expected value
+     * @param delete    data to delete if check succeeds
+     * @return true if the new delete was executed, false otherwise
+     * @throws IOException e
+     */
+    @Override
+    public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, CompareFilter.CompareOp compareOp, byte[] value, Delete delete) throws IOException {
+        return false;
+    }
+
+    @Override
+    public void mutateRow(RowMutations rowMutations) throws IOException {
+        throw new UnsupportedOperationException();
+
+    }
+
+    @Override
+    public Result append(Append append) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Result increment(Increment increment) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long incrementColumnValue(byte[] bytes, byte[] bytes1, byte[] bytes2, long l) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long incrementColumnValue(byte[] bytes, byte[] bytes1, byte[] bytes2, long l, Durability durability) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * @param bytes
+     * @param bytes1
+     * @param bytes2
+     * @param l
+     * @param b
+     * @deprecated
+     */
+    @Override
+    public long incrementColumnValue(byte[] bytes, byte[] bytes1, byte[] bytes2, long l, boolean b) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean isAutoFlush() {
+        return autoflush;
+    }
+
+    @Override
+    public void flushCommits() throws IOException {
+
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+
+    @Override
+    public CoprocessorRpcChannel coprocessorService(byte[] bytes) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public <T extends Service, R> Map<byte[], R> coprocessorService(Class<T> aClass, byte[] bytes, byte[] bytes1, Batch.Call<T, R> call) throws ServiceException, Throwable {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public <T extends Service, R> void coprocessorService(Class<T> aClass, byte[] bytes, byte[] bytes1, Batch.Call<T, R> call, Batch.Callback<R> callback) throws ServiceException, Throwable {
+        throw new UnsupportedOperationException();
+    }
+
+    boolean autoflush = true;
+
+    /**
+     * @param b
+     * @deprecated
+     */
+    @Override
+    public void setAutoFlush(boolean b) {
+        autoflush = b;
+    }
+
+    @Override
+    public void setAutoFlush(boolean b, boolean b1) {
+        autoflush = b;
+    }
+
+    @Override
+    public void setAutoFlushTo(boolean b) {
+        autoflush = b;
+    }
+
+    long writeBufferSize = 0;
+    @Override
+    public long getWriteBufferSize() {
+        return writeBufferSize;
+    }
+
+    @Override
+    public void setWriteBufferSize(long l) throws IOException {
+        writeBufferSize = l;
+    }
+
+    @Override
+    public <R extends Message> Map<byte[], R> batchCoprocessorService(Descriptors.MethodDescriptor methodDescriptor, Message message, byte[] bytes, byte[] bytes1, R r) throws ServiceException, Throwable {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public <R extends Message> void batchCoprocessorService(Descriptors.MethodDescriptor methodDescriptor, Message message, byte[] bytes, byte[] bytes1, R r, Batch.Callback<R> callback) throws ServiceException, Throwable {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Atomically checks if a row/family/qualifier value matches the expected value.
+     * If it does, it performs the row mutations.  If the passed value is null, the check
+     * is for the lack of column (ie: non-existence)
+     *
+     * @param row       to check
+     * @param family    column family to check
+     * @param qualifier column qualifier to check
+     * @param compareOp the comparison operator
+     * @param value     the expected value
+     * @param mutation  mutations to perform if check succeeds
+     * @return true if the new put was executed, false otherwise
+     * @throws IOException e
+     */
+    @Override
+    public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, CompareFilter.CompareOp compareOp, byte[] value, RowMutations mutation) throws IOException {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Topologies/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/pom.xml b/metron-streaming/Metron-Topologies/pom.xml
index c36c325..9a7d8df 100644
--- a/metron-streaming/Metron-Topologies/pom.xml
+++ b/metron-streaming/Metron-Topologies/pom.xml
@@ -42,6 +42,12 @@
         </dependency>
         <dependency>
             <groupId>org.apache.metron</groupId>
+            <artifactId>Metron-Testing</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
             <artifactId>Metron-Common</artifactId>
             <version>${project.parent.version}</version>
         </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/pcap/PcapIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/pcap/PcapIntegrationTest.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/pcap/PcapIntegrationTest.java
index 0aba22f..3337855 100644
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/pcap/PcapIntegrationTest.java
+++ b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/pcap/PcapIntegrationTest.java
@@ -21,9 +21,14 @@ import com.google.common.base.Function;
 import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
 import com.google.common.collect.Iterables;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.hbase.HTableProvider;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelValue;
 import org.apache.metron.integration.util.UnitTestHelper;
 import org.apache.metron.integration.util.integration.ComponentRunner;
 import org.apache.metron.integration.util.integration.Processor;
@@ -33,18 +38,16 @@ import org.apache.metron.integration.util.integration.components.FluxTopologyCom
 import org.apache.metron.integration.util.mock.MockHTable;
 import org.apache.metron.integration.util.threatintel.ThreatIntelHelper;
 import org.apache.metron.parsing.parsers.PcapParser;
+import org.apache.metron.reference.lookup.LookupKV;
 import org.apache.metron.test.converters.HexStringConverter;
-import org.apache.metron.threatintel.ThreatIntelKey;
+import org.apache.metron.hbase.converters.threatintel.ThreatIntelKey;
 import org.apache.metron.threatintel.ThreatIntelResults;
 import org.json.simple.JSONObject;
 import org.junit.Assert;
 import org.junit.Test;
 
 import javax.annotation.Nullable;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
+import java.io.*;
 import java.text.SimpleDateFormat;
 import java.util.*;
 
@@ -53,6 +56,14 @@ public class PcapIntegrationTest {
     private String topologiesDir = "src/main/resources/Metron_Configs/topologies";
     private String targetDir = "target";
 
+    public static class Provider implements TableProvider, Serializable{
+
+        MockHTable.Provider  provider = new MockHTable.Provider();
+        @Override
+        public HTableInterface getTable(Configuration config, String tableName) throws IOException {
+            return provider.getTable(config, tableName);
+        }
+    }
 
     @Test
     public void testTopology() throws Exception {
@@ -107,7 +118,7 @@ public class PcapIntegrationTest {
             setProperty("bolt.hbase.write.buffer.size.in.bytes", "2000000");
             setProperty("bolt.hbase.durability", "SKIP_WAL");
             setProperty("bolt.hbase.partitioner.region.info.refresh.interval.mins","60");
-            setProperty("hbase.provider.impl","" + MockHTable.Provider.class.getName());
+            setProperty("hbase.provider.impl","" + Provider.class.getName());
             setProperty("threat.intel.tracker.table", trackerHBaseTable);
             setProperty("threat.intel.tracker.cf", cf);
             setProperty("threat.intel.ip.table", ipThreatIntelTable);
@@ -121,9 +132,9 @@ public class PcapIntegrationTest {
         //create MockHBaseTables
         final MockHTable trackerTable = (MockHTable)MockHTable.Provider.addToCache(trackerHBaseTable, cf);
         final MockHTable ipTable = (MockHTable)MockHTable.Provider.addToCache(ipThreatIntelTable, cf);
-        ThreatIntelHelper.INSTANCE.load(ipTable, cf, new ArrayList<ThreatIntelResults>(){{
-            add(new ThreatIntelResults(new ThreatIntelKey("10.0.2.3"), new HashMap<String, String>()));
-        }}, 0L);
+        ThreatIntelHelper.INSTANCE.load(ipTable, cf, new ArrayList<LookupKV<ThreatIntelKey, ThreatIntelValue>>(){{
+            add(new LookupKV<>(new ThreatIntelKey("10.0.2.3"), new ThreatIntelValue(new HashMap<String, String>())));
+        }});
         final MockHTable pcapTable = (MockHTable) MockHTable.Provider.addToCache("pcap_test", "t");
         FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder()
                                                                        .withTopologyLocation(new File(topologiesDir + "/pcap/local.yaml"))

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/UnitTestHelper.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/UnitTestHelper.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/UnitTestHelper.java
deleted file mode 100644
index 7af2212..0000000
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/UnitTestHelper.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.util;
-
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
-import org.junit.Assert;
-
-import java.io.File;
-import java.util.Set;
-import java.util.Stack;
-
-public class UnitTestHelper {
-    public static String findDir(String name) {
-        return findDir(new File("."), name);
-    }
-
-    public static String findDir(File startDir, String name) {
-        Stack<File> s = new Stack<File>();
-        s.push(startDir);
-        while(!s.empty()) {
-            File parent = s.pop();
-            if(parent.getName().equalsIgnoreCase(name)) {
-                return parent.getAbsolutePath();
-            }
-            else {
-                File[] children = parent.listFiles();
-                if(children != null) {
-                    for (File child : children) {
-                        s.push(child);
-                    }
-                }
-            }
-        }
-        return null;
-    }
-
-    public static <T> void assertSetEqual(String type, Set<T> expectedPcapIds, Set<T> found) {
-        boolean mismatch = false;
-        for(T f : found) {
-            if(!expectedPcapIds.contains(f)) {
-                mismatch = true;
-                System.out.println("Found " + type + " that I did not expect: " + f);
-            }
-        }
-        for(T expectedId : expectedPcapIds) {
-            if(!found.contains(expectedId)) {
-                mismatch = true;
-                System.out.println("Expected " + type + " that I did not index: " + expectedId);
-            }
-        }
-        Assert.assertFalse(mismatch);
-    }
-
-    public static void verboseLogging() {
-        verboseLogging("%d [%p|%c|%C{1}] %m%n", Level.ALL);
-    }
-    public static void verboseLogging(String pattern, Level level) {
-        ConsoleAppender console = new ConsoleAppender(); //create appender
-        //configure the appender
-        console.setLayout(new PatternLayout(pattern));
-        console.setThreshold(level);
-        console.activateOptions();
-        //add appender to any Logger (here is root)
-        Logger.getRootLogger().addAppender(console);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/ComponentRunner.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/ComponentRunner.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/ComponentRunner.java
deleted file mode 100644
index 18c06f2..0000000
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/ComponentRunner.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.util.integration;
-
-import backtype.storm.utils.Utils;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-public class ComponentRunner {
-    public static class Builder {
-        LinkedHashMap<String, InMemoryComponent> components;
-        String[] startupOrder;
-        String[] shutdownOrder;
-        public Builder() {
-            components = new LinkedHashMap<String, InMemoryComponent>();
-        }
-
-        public Builder withComponent(String name, InMemoryComponent component) {
-            components.put(name, component);
-            return this;
-        }
-
-        public Builder withCustomStartupOrder(String[] startupOrder) {
-            this.startupOrder = startupOrder;
-            return this;
-        }
-        public Builder withCustomShutdownOrder(String[] shutdownOrder) {
-            this.shutdownOrder = shutdownOrder;
-            return this;
-        }
-        private static String[] toOrderedList(Map<String, InMemoryComponent> components) {
-            String[] ret = new String[components.size()];
-            int i = 0;
-            for(String component : components.keySet()) {
-                ret[i++] = component;
-            }
-            return ret;
-        }
-        public ComponentRunner build() {
-            if(shutdownOrder == null) {
-                shutdownOrder = toOrderedList(components);
-            }
-            if(startupOrder == null) {
-                startupOrder = toOrderedList(components);
-            }
-            return new ComponentRunner(components, startupOrder, shutdownOrder);
-        }
-
-    }
-
-    LinkedHashMap<String, InMemoryComponent> components;
-    String[] startupOrder;
-    String[] shutdownOrder;
-    public ComponentRunner( LinkedHashMap<String, InMemoryComponent> components
-                          , String[] startupOrder
-                          , String[] shutdownOrder
-                          )
-    {
-        this.components = components;
-        this.startupOrder = startupOrder;
-        this.shutdownOrder = shutdownOrder;
-
-    }
-
-    public <T extends InMemoryComponent> T getComponent(String name, Class<T> clazz) {
-        return clazz.cast(getComponents().get(name));
-    }
-
-    public LinkedHashMap<String, InMemoryComponent> getComponents() {
-        return components;
-    }
-
-    public void start() throws UnableToStartException {
-        for(String componentName : startupOrder) {
-            components.get(componentName).start();
-        }
-    }
-    public void stop() {
-        for(String componentName : shutdownOrder) {
-            components.get(componentName).stop();
-        }
-    }
-
-    public <T> T process(Processor<T> successState) {
-        return process(successState, 5, 30000, 120000);
-    }
-
-    public <T> T process(Processor<T> successState, int numRetries, long timeBetweenAttempts, long maxTimeMs) {
-        int retryCount = 0;
-        long start = System.currentTimeMillis();
-        while(true) {
-            long duration = System.currentTimeMillis() - start;
-            if(duration > maxTimeMs) {
-                throw new RuntimeException("Took too long to complete: " + duration + " > " + maxTimeMs);
-            }
-            ReadinessState state = successState.process(this);
-            if(state == ReadinessState.READY) {
-                return successState.getResult();
-            }
-            else if(state == ReadinessState.NOT_READY) {
-                retryCount++;
-                if(numRetries > 0 && retryCount > numRetries) {
-                    throw new RuntimeException("Too many retries: " + retryCount);
-                }
-            }
-            Utils.sleep(timeBetweenAttempts);
-        }
-    }
-
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/InMemoryComponent.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/InMemoryComponent.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/InMemoryComponent.java
deleted file mode 100644
index b5224a2..0000000
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/InMemoryComponent.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.util.integration;
-
-public interface InMemoryComponent {
-    public void start() throws UnableToStartException;
-    public void stop();
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/Processor.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/Processor.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/Processor.java
deleted file mode 100644
index 59f3ece..0000000
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/Processor.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.util.integration;
-
-public interface Processor<T> {
-    ReadinessState process(ComponentRunner runner);
-    T getResult();
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/ReadinessState.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/ReadinessState.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/ReadinessState.java
deleted file mode 100644
index 59ce69a..0000000
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/ReadinessState.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.util.integration;
-
-public enum ReadinessState {
-    READY, NOT_READY;
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0e1055aa/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/UnableToStartException.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/UnableToStartException.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/UnableToStartException.java
deleted file mode 100644
index 4e691f4..0000000
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/integration/UnableToStartException.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.util.integration;
-
-public class UnableToStartException extends Exception {
-    public UnableToStartException(String message) {
-        super(message);
-    }
-    public UnableToStartException(String message, Throwable t) {
-        super(message, t);
-    }
-}