You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/07/20 13:14:24 UTC
[1/5] incubator-metron git commit: METRON-154: Decouple enrichment
and indexing closes apache/incubator-metron#192
Repository: incubator-metron
Updated Branches:
refs/heads/master 1c568bf04 -> 5ffcef8d4
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
new file mode 100644
index 0000000..0225137
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
+import org.apache.hadoop.hdfs.util.MD5FileUtils;
+import org.apache.hadoop.io.MD5Hash;
+import org.apache.log4j.Logger;
+import org.apache.storm.hdfs.bolt.format.FileNameFormat;
+import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
+import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy;
+import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
+import org.apache.storm.hdfs.common.rotation.RotationAction;
+import org.apache.storm.hdfs.common.security.HdfsSecurityUtil;
+import org.json.simple.JSONObject;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.*;
+
+public class SourceHandler {
+ private static final Logger LOG = Logger.getLogger(SourceHandler.class);
+ List<RotationAction> rotationActions = new ArrayList<>();
+ FileRotationPolicy rotationPolicy;
+ SyncPolicy syncPolicy;
+ FileNameFormat fileNameFormat;
+ private long offset = 0;
+ private int rotation = 0;
+ private transient FSDataOutputStream out;
+ private transient Object writeLock;
+ protected transient Timer rotationTimer; // only used for TimedRotationPolicy
+ protected transient FileSystem fs;
+ protected transient Path currentFile;
+ public SourceHandler(List<RotationAction> rotationActions
+ , FileRotationPolicy rotationPolicy
+ , SyncPolicy syncPolicy
+ , FileNameFormat fileNameFormat
+ , Map config
+ ) throws IOException {
+ this.rotationActions = rotationActions;
+ this.rotationPolicy = rotationPolicy;
+ this.syncPolicy = syncPolicy;
+ this.fileNameFormat = fileNameFormat;
+ initialize(config);
+ }
+
+ public void handle(List<JSONObject> messages) throws Exception{
+
+ for(JSONObject message : messages) {
+ byte[] bytes = (message.toJSONString() + "\n").getBytes();
+ synchronized (this.writeLock) {
+ out.write(bytes);
+ this.offset += bytes.length;
+
+ if (this.syncPolicy.mark(null, this.offset)) {
+ if (this.out instanceof HdfsDataOutputStream) {
+ ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
+ } else {
+ this.out.hsync();
+ }
+ this.syncPolicy.reset();
+ }
+ }
+
+ if (this.rotationPolicy.mark(null, this.offset)) {
+ rotateOutputFile(); // synchronized
+ this.offset = 0;
+ this.rotationPolicy.reset();
+ }
+ }
+ }
+
+ private void initialize(Map config) throws IOException {
+ this.writeLock = new Object();
+ Configuration hdfsConfig = new Configuration();
+ this.fs = FileSystem.get(new Configuration());
+ HdfsSecurityUtil.login(config, hdfsConfig);
+ this.currentFile = createOutputFile();
+ if(this.rotationPolicy instanceof TimedRotationPolicy){
+ long interval = ((TimedRotationPolicy)this.rotationPolicy).getInterval();
+ this.rotationTimer = new Timer(true);
+ TimerTask task = new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ rotateOutputFile();
+ } catch(IOException e){
+ LOG.warn("IOException during scheduled file rotation.", e);
+ }
+ }
+ };
+ this.rotationTimer.scheduleAtFixedRate(task, interval, interval);
+ }
+ }
+
+ protected void rotateOutputFile() throws IOException {
+ LOG.info("Rotating output file...");
+ long start = System.currentTimeMillis();
+ synchronized (this.writeLock) {
+ closeOutputFile();
+ this.rotation++;
+
+ Path newFile = createOutputFile();
+ LOG.info("Performing " + this.rotationActions.size() + " file rotation actions." );
+ for (RotationAction action : this.rotationActions) {
+ action.execute(this.fs, this.currentFile);
+ }
+ this.currentFile = newFile;
+ }
+ long time = System.currentTimeMillis() - start;
+ LOG.info("File rotation took " + time + " ms.");
+ }
+
+ private Path createOutputFile() throws IOException {
+ Path path = new Path(this.fileNameFormat.getPath(), this.fileNameFormat.getName(this.rotation, System.currentTimeMillis()));
+ if(fs.getScheme().equals("file")) {
+ //in the situation where we're running this in a local filesystem, flushing doesn't work.
+ fs.mkdirs(path.getParent());
+ this.out = new FSDataOutputStream(new FileOutputStream(path.toString()), null);
+ }
+ else {
+ this.out = this.fs.create(path);
+ }
+ return path;
+ }
+
+ private void closeOutputFile() throws IOException {
+ this.out.close();
+ }
+
+
+ public void close() {
+ try {
+ closeOutputFile();
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to close output file.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java
new file mode 100644
index 0000000..f6acf6f
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer.kafka;
+
+import backtype.storm.tuple.Tuple;
+import com.google.common.base.Joiner;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.MessageWriter;
+import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.common.utils.KafkaUtils;
+import org.apache.metron.common.utils.StringUtils;
+import org.apache.metron.writer.AbstractWriter;
+import org.json.simple.JSONObject;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+public class KafkaWriter extends AbstractWriter implements MessageWriter<JSONObject>, Serializable {
+ public enum Configurations {
+ BROKER("kafka.brokerUrl")
+ ,KEY_SERIALIZER("kafka.keySerializer")
+ ,ZK_QUORUM("kafka.zkQuorum")
+ ,VALUE_SERIALIZER("kafka.valueSerializer")
+ ,REQUIRED_ACKS("kafka.requiredAcks")
+ ,TOPIC("kafka.topic")
+ ,PRODUCER_CONFIGS("kafka.producerConfigs");
+ ;
+ String key;
+ Configurations(String key) {
+ this.key = key;
+ }
+ public Object get(Optional<String> configPrefix, Map<String, Object> config) {
+ return config.get(StringUtils.join(".", configPrefix, Optional.of(key)));
+ }
+ public <T> T getAndConvert(Optional<String> configPrefix, Map<String, Object> config, Class<T> clazz) {
+ Object o = get(configPrefix, config);
+ if(o != null) {
+ return ConversionUtils.convert(o, clazz);
+ }
+ return null;
+ }
+ }
+ private String brokerUrl;
+ private String keySerializer = "org.apache.kafka.common.serialization.StringSerializer";
+ private String valueSerializer = "org.apache.kafka.common.serialization.StringSerializer";
+ private int requiredAcks = 1;
+ private String kafkaTopic = Constants.ENRICHMENT_TOPIC;
+ private KafkaProducer kafkaProducer;
+ private String configPrefix = null;
+ private String zkQuorum = null;
+ private Map<String, Object> producerConfigs = new HashMap<>();
+
+ public KafkaWriter() {}
+
+ public KafkaWriter(String brokerUrl) {
+ this.brokerUrl = brokerUrl;
+ }
+
+ public KafkaWriter withZkQuorum(String zkQuorum) {
+ this.zkQuorum = zkQuorum;
+ return this;
+ }
+ public KafkaWriter withKeySerializer(String keySerializer) {
+ this.keySerializer = keySerializer;
+ return this;
+ }
+
+ public KafkaWriter withValueSerializer(String valueSerializer) {
+ this.valueSerializer = valueSerializer;
+ return this;
+ }
+
+ public KafkaWriter withRequiredAcks(Integer requiredAcks) {
+ this.requiredAcks = requiredAcks;
+ return this;
+ }
+
+ public KafkaWriter withTopic(String topic) {
+ this.kafkaTopic= topic;
+ return this;
+ }
+ public KafkaWriter withConfigPrefix(String prefix) {
+ this.configPrefix = prefix;
+ return this;
+ }
+
+ public KafkaWriter withProducerConfigs(Map<String, Object> extraConfigs) {
+ this.producerConfigs = extraConfigs;
+ return this;
+ }
+
+ public Optional<String> getConfigPrefix() {
+ return Optional.ofNullable(configPrefix);
+ }
+
+ @Override
+ public void configure(String sensorName, WriterConfiguration configuration) {
+ Map<String, Object> configMap = configuration.getSensorConfig(sensorName);
+ String brokerUrl = Configurations.BROKER.getAndConvert(getConfigPrefix(), configMap, String.class);
+ if(brokerUrl != null) {
+ this.brokerUrl = brokerUrl;
+ }
+ String zkQuorum = Configurations.ZK_QUORUM.getAndConvert(getConfigPrefix(), configMap, String.class);
+ if(zkQuorum != null) {
+ withZkQuorum(zkQuorum);
+ }
+ String keySerializer = Configurations.KEY_SERIALIZER.getAndConvert(getConfigPrefix(), configMap, String.class);
+ if(keySerializer != null) {
+ withKeySerializer(keySerializer);
+ }
+ String valueSerializer = Configurations.VALUE_SERIALIZER.getAndConvert(getConfigPrefix(), configMap, String.class);
+ if(valueSerializer != null) {
+ withValueSerializer(keySerializer);
+ }
+ Integer requiredAcks = Configurations.REQUIRED_ACKS.getAndConvert(getConfigPrefix(), configMap, Integer.class);
+ if(requiredAcks!= null) {
+ withRequiredAcks(requiredAcks);
+ }
+ String topic = Configurations.TOPIC.getAndConvert(getConfigPrefix(), configMap, String.class);
+ if(topic != null) {
+ withTopic(topic);
+ }
+ Map<String, Object> producerConfigs = (Map)Configurations.PRODUCER_CONFIGS.get(getConfigPrefix(), configMap);
+ if(producerConfigs != null) {
+ withProducerConfigs(producerConfigs);
+ }
+ }
+
+ public Map<String, Object> createProducerConfigs() {
+ Map<String, Object> producerConfig = new HashMap<>();
+ producerConfig.put("bootstrap.servers", brokerUrl);
+ producerConfig.put("key.serializer", keySerializer);
+ producerConfig.put("value.serializer", valueSerializer);
+ producerConfig.put("request.required.acks", requiredAcks);
+ producerConfig.putAll(producerConfigs == null?new HashMap<>():producerConfigs);
+ return producerConfig;
+ }
+
+ @Override
+ public void init() {
+ if(this.zkQuorum != null && this.brokerUrl == null) {
+ try {
+ this.brokerUrl = Joiner.on(",").join(KafkaUtils.INSTANCE.getBrokersFromZookeeper(this.zkQuorum));
+ } catch (Exception e) {
+ throw new IllegalStateException("Cannot read kafka brokers from zookeeper and you didn't specify them, giving up!", e);
+ }
+ }
+ this.kafkaProducer = new KafkaProducer<>(createProducerConfigs());
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void write(String sourceType, WriterConfiguration configurations, Tuple tuple, JSONObject message) throws Exception {
+ kafkaProducer.send(new ProducerRecord<String, String>(kafkaTopic, message.toJSONString()));
+ }
+
+ @Override
+ public void close() throws Exception {
+ kafkaProducer.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java
new file mode 100644
index 0000000..49bab6f
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetter.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.message;
+
+import backtype.storm.tuple.Tuple;
+import org.json.simple.JSONObject;
+
+public interface MessageGetter {
+ JSONObject getMessage(Tuple t);
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java
new file mode 100644
index 0000000..f25ea62
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/MessageGetters.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.message;
+
+
+import backtype.storm.tuple.Tuple;
+import org.json.simple.JSONObject;
+
+public enum MessageGetters implements MessageGetter{
+ RAW(RawMessageGetter.DEFAULT)
+ ,NAMED(NamedMessageGetter.DEFAULT)
+ ;
+ MessageGetter getter;
+ MessageGetters(MessageGetter getter) {
+ this.getter = getter;
+ }
+ @Override
+ public JSONObject getMessage(Tuple t) {
+ return getter.getMessage(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java
new file mode 100644
index 0000000..70cfab2
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/NamedMessageGetter.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.message;
+
+import backtype.storm.tuple.Tuple;
+import org.json.simple.JSONObject;
+
+public class NamedMessageGetter implements MessageGetter {
+ public static NamedMessageGetter DEFAULT = new NamedMessageGetter("message");
+ private String messageName;
+ public NamedMessageGetter(String name) {
+ this.messageName = name;
+ }
+ @Override
+ public JSONObject getMessage(Tuple tuple) {
+ return (JSONObject)tuple.getValueByField(messageName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java
new file mode 100644
index 0000000..16c0173
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/message/RawMessageGetter.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.message;
+
+import backtype.storm.tuple.Tuple;
+import org.apache.metron.common.utils.JSONUtils;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+
+import java.io.UnsupportedEncodingException;
+
+public class RawMessageGetter implements MessageGetter {
+ public static RawMessageGetter DEFAULT = new RawMessageGetter(0);
+ private ThreadLocal<JSONParser> parser = new ThreadLocal<JSONParser>() {
+ @Override
+ protected JSONParser initialValue() {
+ return new JSONParser();
+ }
+ };
+ int position = 0;
+ public RawMessageGetter(int position) {
+ this.position = position;
+ }
+ @Override
+ public JSONObject getMessage(Tuple t) {
+ byte[] data = t.getBinary(position);
+ try {
+ return (JSONObject) parser.get().parse(new String(data, "UTF8"));
+ } catch (Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/NoopWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/NoopWriterTest.java b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/NoopWriterTest.java
new file mode 100644
index 0000000..3a5a762
--- /dev/null
+++ b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/NoopWriterTest.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class NoopWriterTest {
+ @Test
+ public void testFixedLatencyConfig() {
+ NoopWriter writer = new NoopWriter().withLatency("10");
+ Assert.assertTrue(writer.sleepFunction instanceof NoopWriter.FixedLatency);
+ NoopWriter.FixedLatency sleepFunction = (NoopWriter.FixedLatency)writer.sleepFunction;
+ Assert.assertEquals(10, sleepFunction.getLatency());
+ }
+
+ private void ensureRandomLatencyConfig(String latencyConfig, int min, int max) {
+ NoopWriter writer = new NoopWriter().withLatency(latencyConfig);
+ Assert.assertTrue(writer.sleepFunction instanceof NoopWriter.RandomLatency);
+ NoopWriter.RandomLatency sleepFunction = (NoopWriter.RandomLatency)writer.sleepFunction;
+ Assert.assertEquals(min, sleepFunction.getMin());
+ Assert.assertEquals(max, sleepFunction.getMax());
+ }
+
+ @Test
+ public void testRandomLatencyConfig() {
+ ensureRandomLatencyConfig("10,20", 10, 20);
+ ensureRandomLatencyConfig("10, 20", 10, 20);
+ ensureRandomLatencyConfig("10 ,20", 10, 20);
+ ensureRandomLatencyConfig("10 , 20", 10, 20);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/kafka/KafkaWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/kafka/KafkaWriterTest.java b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/kafka/KafkaWriterTest.java
new file mode 100644
index 0000000..1b95430
--- /dev/null
+++ b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/kafka/KafkaWriterTest.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.kafka;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class KafkaWriterTest {
+
+
+ public static final String SENSOR_TYPE = "test";
+ public WriterConfiguration createConfiguration(final Map<String, Object> parserConfig) {
+ ParserConfigurations configurations = new ParserConfigurations();
+ configurations.updateSensorParserConfig( SENSOR_TYPE
+ , new SensorParserConfig() {{
+ setParserConfig(parserConfig);
+ }}
+ );
+ return new ParserWriterConfiguration(configurations);
+ }
+
+ @Test
+ public void testHappyPathGlobalConfig() throws Exception {
+ KafkaWriter writer = new KafkaWriter();
+ WriterConfiguration configuration = createConfiguration(
+ new HashMap<String, Object>() {{
+ put("kafka.brokerUrl" , "localhost:6667");
+ put("kafka.topic" , SENSOR_TYPE);
+ put("kafka.producerConfigs" , ImmutableMap.of("key1", 1, "key2", "value2"));
+ }}
+ );
+
+ writer.configure(SENSOR_TYPE, configuration);
+ Map<String, Object> producerConfigs = writer.createProducerConfigs();
+ Assert.assertEquals(producerConfigs.get("bootstrap.servers"), "localhost:6667");
+ Assert.assertEquals(producerConfigs.get("key.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
+ Assert.assertEquals(producerConfigs.get("value.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
+ Assert.assertEquals(producerConfigs.get("request.required.acks"), 1);
+ Assert.assertEquals(producerConfigs.get("key1"), 1);
+ Assert.assertEquals(producerConfigs.get("key2"), "value2");
+ }
+
+ @Test
+ public void testHappyPathGlobalConfigWithPrefix() throws Exception {
+ KafkaWriter writer = new KafkaWriter();
+ writer.withConfigPrefix("prefix");
+ WriterConfiguration configuration = createConfiguration(
+ new HashMap<String, Object>() {{
+ put("prefix.kafka.brokerUrl" , "localhost:6667");
+ put("prefix.kafka.topic" , SENSOR_TYPE);
+ put("prefix.kafka.producerConfigs" , ImmutableMap.of("key1", 1, "key2", "value2"));
+ }}
+ );
+
+ writer.configure(SENSOR_TYPE, configuration);
+ Map<String, Object> producerConfigs = writer.createProducerConfigs();
+ Assert.assertEquals(producerConfigs.get("bootstrap.servers"), "localhost:6667");
+ Assert.assertEquals(producerConfigs.get("key.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
+ Assert.assertEquals(producerConfigs.get("value.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
+ Assert.assertEquals(producerConfigs.get("request.required.acks"), 1);
+ Assert.assertEquals(producerConfigs.get("key1"), 1);
+ Assert.assertEquals(producerConfigs.get("key2"), "value2");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/pom.xml b/metron-platform/pom.xml
index ec274f0..1f8d5f1 100644
--- a/metron-platform/pom.xml
+++ b/metron-platform/pom.xml
@@ -51,6 +51,8 @@
<module>metron-integration-test</module>
<module>metron-test-utilities</module>
<module>metron-api</module>
+ <module>metron-indexing</module>
+ <module>metron-writer</module>
<module>metron-hbase</module>
<module>elasticsearch-shaded</module>
<module>metron-elasticsearch</module>
[5/5] incubator-metron git commit: METRON-154: Decouple enrichment
and indexing closes apache/incubator-metron#192
Posted by ce...@apache.org.
METRON-154: Decouple enrichment and indexing closes apache/incubator-metron#192
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/5ffcef8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/5ffcef8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/5ffcef8d
Branch: refs/heads/master
Commit: 5ffcef8d4a9ec0788dca094f54b07d99c44c6bef
Parents: 1c568bf
Author: cstella <ce...@gmail.com>
Authored: Wed Jul 20 09:14:16 2016 -0400
Committer: cstella <ce...@gmail.com>
Committed: Wed Jul 20 09:14:16 2016 -0400
----------------------------------------------------------------------
metron-deployment/amazon-ec2/conf/defaults.yml | 1 +
.../inventory/full-dev-platform/group_vars/all | 3 +-
.../inventory/metron_example/group_vars/all | 1 +
.../roles/metron_kafka_topics/defaults/main.yml | 12 +-
.../roles/metron_streaming/defaults/main.yml | 4 +
.../metron_streaming/tasks/copy_bundles.yml | 10 +
.../roles/metron_streaming/tasks/topologies.yml | 28 +-
.../roles/monit/tasks/monit-definitions.yml | 8 +-
metron-deployment/roles/monit/tasks/scripts.yml | 2 +
.../monit/enrichment-elasticsearch.monit | 26 -
.../monit/templates/monit/enrichment-solr.monit | 26 -
.../monit/templates/monit/enrichment.monit | 26 +
.../monit/indexing-elasticsearch.monit | 26 +
.../monit/templates/monit/indexing-solr.monit | 26 +
metron-platform/metron-common/pom.xml | 4 +
.../org/apache/metron/common/Constants.java | 2 +
.../apache/metron/common/utils/BloomFilter.java | 63 +++
.../apache/metron/common/utils/KafkaUtils.java | 54 ++
.../metron/common/writer/AbstractWriter.java | 26 -
.../common/writer/BulkWriterComponent.java | 130 -----
.../apache/metron/common/writer/NoopWriter.java | 143 ------
.../common/writer/WriterToBulkWriter.java | 54 --
.../metron/common/writer/NoopWriterTest.java | 48 --
metron-platform/metron-elasticsearch/pom.xml | 29 ++
.../src/main/config/elasticsearch.properties | 55 +--
.../scripts/start_elasticsearch_topology.sh | 2 +-
.../ElasticsearchEnrichmentIntegrationTest.java | 98 ----
.../ElasticsearchIndexingIntegrationTest.java | 100 ++++
metron-platform/metron-enrichment/pom.xml | 72 ++-
.../src/main/assembly/assembly.xml | 8 +
.../src/main/config/enrichment.properties | 82 ++++
.../src/main/flux/enrichment/remote.yaml | 69 +--
.../src/main/flux/enrichment/test.yaml | 64 +--
.../enrichment/bolt/BulkMessageWriterBolt.java | 83 ----
.../accesstracker/BloomAccessTracker.java | 32 +-
.../writer/SimpleHbaseEnrichmentWriter.java | 275 +++++++++++
.../hbase/SimpleHbaseEnrichmentWriter.java | 282 -----------
.../apache/metron/writer/hdfs/HdfsWriter.java | 96 ----
.../writer/hdfs/SourceAwareMoveAction.java | 48 --
.../writer/hdfs/SourceFileNameFormat.java | 48 --
.../metron/writer/hdfs/SourceHandler.java | 160 ------
.../main/scripts/start_enrichment_topology.sh | 22 +
.../bolt/BulkMessageWriterBoltTest.java | 3 +-
.../integration/EnrichmentIntegrationTest.java | 426 ++++++++++++++++
.../components/ConfigUploadComponent.java | 114 +++++
.../integration/mock/MockGeoAdapter.java | 64 +++
.../integration/mock/MockHBaseConnector.java | 52 ++
.../integration/mock/MockTableProvider.java | 45 ++
.../integration/utils/SampleUtil.java | 57 +++
metron-platform/metron-indexing/pom.xml | 253 ++++++++++
.../src/main/assembly/assembly.xml | 57 +++
.../src/main/flux/indexing/remote.yaml | 152 ++++++
.../integration/IndexingIntegrationTest.java | 240 +++++++++
metron-platform/metron-integration-test/pom.xml | 5 -
.../metron/integration/BaseIntegrationTest.java | 1 -
.../integration/EnrichmentIntegrationTest.java | 489 -------------------
.../components/ConfigUploadComponent.java | 114 -----
.../metron/integration/mock/MockGeoAdapter.java | 64 ---
.../integration/mock/MockHBaseConnector.java | 52 --
.../integration/mock/MockTableProvider.java | 45 --
.../metron/integration/utils/SampleUtil.java | 57 ---
metron-platform/metron-parsers/pom.xml | 16 +-
.../src/main/flux/asa/remote.yaml | 2 +-
.../metron-parsers/src/main/flux/asa/test.yaml | 2 +-
.../src/main/flux/fireeye/remote.yaml | 2 +-
.../src/main/flux/fireeye/test.yaml | 2 +-
.../src/main/flux/ise/remote.yaml | 2 +-
.../metron-parsers/src/main/flux/ise/test.yaml | 2 +-
.../src/main/flux/lancope/remote.yaml | 2 +-
.../src/main/flux/lancope/test.yaml | 2 +-
.../src/main/flux/paloalto/remote.yaml | 2 +-
.../src/main/flux/paloalto/test.yaml | 2 +-
.../src/main/flux/sourcefire/remote.yaml | 2 +-
.../src/main/flux/sourcefire/test.yaml | 2 +-
.../src/main/flux/websphere/remote.yaml | 2 +-
.../src/main/flux/websphere/test.yaml | 2 +-
.../apache/metron/parsers/bolt/ParserBolt.java | 10 -
.../metron/parsers/bolt/WriterHandler.java | 4 +-
.../parsers/topology/ParserTopologyBuilder.java | 4 +-
.../metron/parsers/writer/KafkaWriter.java | 164 -------
.../integration/ParserIntegrationTest.java | 6 +-
.../metron/parsers/writer/KafkaWriterTest.java | 95 ----
.../SimpleHBaseEnrichmentWriterTest.java | 5 +-
...pleHbaseEnrichmentWriterIntegrationTest.java | 14 +-
.../integration/WriterBoltIntegrationTest.java | 17 +-
metron-platform/metron-solr/pom.xml | 50 +-
.../metron-solr/src/main/config/solr.properties | 61 +--
.../src/main/scripts/start_solr_topology.sh | 2 +-
.../SolrEnrichmentIntegrationTest.java | 114 -----
.../SolrIndexingIntegrationTest.java | 114 +++++
.../metron/solr/writer/SolrWriterTest.java | 4 +-
metron-platform/metron-writer/pom.xml | 231 +++++++++
.../apache/metron/writer/AbstractWriter.java | 26 +
.../metron/writer/BulkWriterComponent.java | 127 +++++
.../org/apache/metron/writer/NoopWriter.java | 141 ++++++
.../metron/writer/WriterToBulkWriter.java | 58 +++
.../writer/bolt/BulkMessageWriterBolt.java | 114 +++++
.../apache/metron/writer/hdfs/HdfsWriter.java | 96 ++++
.../writer/hdfs/SourceAwareMoveAction.java | 48 ++
.../writer/hdfs/SourceFileNameFormat.java | 48 ++
.../metron/writer/hdfs/SourceHandler.java | 160 ++++++
.../apache/metron/writer/kafka/KafkaWriter.java | 181 +++++++
.../metron/writer/message/MessageGetter.java | 26 +
.../metron/writer/message/MessageGetters.java | 37 ++
.../writer/message/NamedMessageGetter.java | 34 ++
.../metron/writer/message/RawMessageGetter.java | 50 ++
.../apache/metron/writer/NoopWriterTest.java | 48 ++
.../metron/writer/kafka/KafkaWriterTest.java | 88 ++++
metron-platform/pom.xml | 2 +
109 files changed, 4044 insertions(+), 2817 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/amazon-ec2/conf/defaults.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/conf/defaults.yml b/metron-deployment/amazon-ec2/conf/defaults.yml
index 2874b2c..9e05bbc 100644
--- a/metron-deployment/amazon-ec2/conf/defaults.yml
+++ b/metron-deployment/amazon-ec2/conf/defaults.yml
@@ -30,6 +30,7 @@ services_to_start:
- bro-parser
- snort-parser
- enrichment
+ - indexing
# the ami for cent6 by region
amis_by_region:
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/inventory/full-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all b/metron-deployment/inventory/full-dev-platform/group_vars/all
index 2e87a29..6afe4ae 100644
--- a/metron-deployment/inventory/full-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/full-dev-platform/group_vars/all
@@ -21,15 +21,14 @@ services_to_start:
- elasticsearch
- pcap-service
- kibana
- - yaf
- snort
- snort-logs
- bro
- pcap-replay
- - yaf-parser
- bro-parser
- snort-parser
- enrichment
+ - indexing
# ambari
ambari_host: "{{ groups.ambari_master[0] }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/metron_example/group_vars/all b/metron-deployment/inventory/metron_example/group_vars/all
index bfa2469..a0d9257 100644
--- a/metron-deployment/inventory/metron_example/group_vars/all
+++ b/metron-deployment/inventory/metron_example/group_vars/all
@@ -30,6 +30,7 @@ services_to_start:
- bro-parser
- snort-parser
- enrichment
+ - indexing
#Ansible Variables
ansible_ssh_private_key_file: /Path/to/private/key/file #Change This
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/metron_kafka_topics/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_kafka_topics/defaults/main.yml b/metron-deployment/roles/metron_kafka_topics/defaults/main.yml
index aac9a06..fa0ee02 100644
--- a/metron-deployment/roles/metron_kafka_topics/defaults/main.yml
+++ b/metron-deployment/roles/metron_kafka_topics/defaults/main.yml
@@ -17,10 +17,12 @@
---
kafka_home: /usr/hdp/current/kafka-broker/
topics_to_create:
- - { topic: "pcap", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
- - { topic: "bro", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
- - { topic: "yaf", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
- - { topic: "snort", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
- - { topic: "enrichments", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "pcap", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "bro", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "yaf", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "snort", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "enrichments", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
- { topic: "parser_invalid", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
- { topic: "parser_error", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "indexing", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
+ - { topic: "indexing_error", num_partitions: 1, replication_factor: 1, retention_gb: 10 }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/defaults/main.yml b/metron-deployment/roles/metron_streaming/defaults/main.yml
index cbec601..8618c38 100644
--- a/metron-deployment/roles/metron_streaming/defaults/main.yml
+++ b/metron-deployment/roles/metron_streaming/defaults/main.yml
@@ -20,11 +20,13 @@ metron_solr_jar_name: metron-solr-{{ metron_version }}.jar
metron_elasticsearch_jar_name: metron-elasticsearch-{{ metron_version }}.jar
metron_pcap_jar_name: metron-pcap-backend-{{ metron_version }}.jar
metron_parsers_jar_name: metron-parsers-{{ metron_version }}.jar
+metron_enrichment_jar_name: metron-enrichment-{{ metron_version }}-uber.jar
metron_common_bundle_name: metron-common-{{ metron_version }}-archive.tar.gz
metron_pcap_bundle_name: metron-pcap-backend-{{ metron_version }}-archive.tar.gz
metron_data_management_bundle_name: metron-data-management-{{ metron_version }}-archive.tar.gz
metron_enrichment_bundle_name: metron-enrichment-{{ metron_version }}-archive.tar.gz
+metron_indexing_bundle_name: metron-indexing-{{ metron_version }}-archive.tar.gz
metron_solr_bundle_name: metron-solr-{{ metron_version }}-archive.tar.gz
metron_elasticsearch_bundle_name: metron-elasticsearch-{{ metron_version }}-archive.tar.gz
metron_parsers_bundle_name: metron-parsers-{{ metron_version }}-archive.tar.gz
@@ -32,6 +34,7 @@ metron_pcap_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-pcap-b
metron_common_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-common/target/{{ metron_common_bundle_name }}"
metron_data_management_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-data-management/target/{{ metron_data_management_bundle_name }}"
metron_enrichment_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-enrichment/target/{{ metron_enrichment_bundle_name }}"
+metron_indexing_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-indexing/target/{{ metron_indexing_bundle_name }}"
metron_solr_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-solr/target/{{ metron_solr_bundle_name }}"
metron_elasticsearch_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-elasticsearch/target/{{ metron_elasticsearch_bundle_name }}"
metron_parsers_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-parsers/target/{{ metron_parsers_bundle_name }}"
@@ -42,6 +45,7 @@ zookeeper_global_config_path: "{{ zookeeper_config_path }}/global.json"
metron_pcap_properties_config_path: "{{ metron_directory }}/config/pcap.properties"
metron_solr_properties_config_path: "{{ metron_directory }}/config/solr.properties"
metron_elasticsearch_properties_config_path: "{{ metron_directory }}/config/elasticsearch.properties"
+metron_enrichment_properties_config_path: "{{ metron_directory }}/config/enrichment.properties"
metron_parsers_properties_config_path: "{{ metron_directory }}/config/parsers.properties"
hbase_config_path: "/etc/hbase/conf"
hdfs_config_path: "/etc/hadoop/conf"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml b/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml
index 97fb259..519b7d3 100644
--- a/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml
+++ b/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml
@@ -30,6 +30,12 @@
src: "{{ metron_enrichment_bundle_path }}"
dest: "{{ metron_directory }}"
+- name: Copy Metron Indexing bundle
+ copy:
+ src: "{{ metron_indexing_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+
- name: Copy Metron Parsers bundle
copy:
src: "{{ metron_parsers_bundle_path }}"
@@ -58,6 +64,7 @@
- tar xzvf metron-solr*.tar.gz
- tar xzvf metron-elasticsearch*.tar.gz
- tar xzvf metron-enrichment*.tar.gz
+ - tar xzvf metron-indexing*.tar.gz
- tar xzvf metron-parsers*.tar.gz
- tar xzvf metron-data-management*.tar.gz
- tar xzvf metron-common*.tar.gz
@@ -67,6 +74,9 @@
- name: Add *-site.xml files to topology jars
shell: "cd {{ item.config_path }} && jar -uf {{ metron_directory }}/lib/{{ item.jar_name }} {{ item.file_name }}"
with_items:
+ - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_enrichment_jar_name }}", file_name: "hbase-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_enrichment_jar_name }}", file_name: "core-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_enrichment_jar_name }}", file_name: "hdfs-site.xml" }
- { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hbase-site.xml" }
- { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "core-site.xml" }
- { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hdfs-site.xml" }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/metron_streaming/tasks/topologies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/topologies.yml b/metron-deployment/roles/metron_streaming/tasks/topologies.yml
index 2010eae..0ca0b3d 100644
--- a/metron-deployment/roles/metron_streaming/tasks/topologies.yml
+++ b/metron-deployment/roles/metron_streaming/tasks/topologies.yml
@@ -29,18 +29,14 @@
- { regexp: "kafka.pcap.out=", line: "kafka.pcap.out={{ pcap_hdfs_path }}" }
- { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" }
-- name: Configure Metron Solr topology
+- name: Configure Metron Enrichment topology
lineinfile: >
- dest={{ metron_solr_properties_config_path }}
+ dest={{ metron_enrichment_properties_config_path }}
regexp="{{ item.regexp }}"
line="{{ item.line }}"
with_items:
- { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
- { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
- - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
- - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
- - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
- - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
- { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
- { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" }
- { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
@@ -50,6 +46,17 @@
- { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" }
- { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
- { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
+
+
+- name: Configure Metron Solr topology
+ lineinfile: >
+ dest={{ metron_solr_properties_config_path }}
+ regexp="{{ item.regexp }}"
+ line="{{ item.line }}"
+ with_items:
+ - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
+ - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
+ - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
- { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
- { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
- { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
@@ -67,15 +74,6 @@
- { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
- { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
- { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
- - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
- - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" }
- - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
- - { regexp: "threat.intel.simple.hbase.table=", line: "threat.intel.simple.hbase.table={{ threatintel_hbase_table }}" }
- - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" }
- - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" }
- - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" }
- - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
- - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
- { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
- { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
- { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/monit/tasks/monit-definitions.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/monit-definitions.yml b/metron-deployment/roles/monit/tasks/monit-definitions.yml
index 3778e14..f9e3897 100644
--- a/metron-deployment/roles/monit/tasks/monit-definitions.yml
+++ b/metron-deployment/roles/monit/tasks/monit-definitions.yml
@@ -24,10 +24,14 @@
template: src=monit/elasticsearch.monit dest={{ monit_config_home }}/elasticsearch.monit
when: ("search" in group_names) and (install_elasticsearch | default(True))
-- name: Create monit definition for enrichment with elasticsearch
- template: src=monit/enrichment-elasticsearch.monit dest={{ monit_config_home }}/enrichment-elasticsearch.monit
+- name: Create monit definition for indexing with elasticsearch
+ template: src=monit/indexing-elasticsearch.monit dest={{ monit_config_home }}/indexing-elasticsearch.monit
when: ("enrichment" in group_names) and (install_elasticsearch | default(True))
+- name: Create monit definition for enrichment
+ template: src=monit/enrichment.monit dest={{ monit_config_home }}/enrichment.monit
+ when: ("enrichment" in group_names)
+
- name: Create monit definition for kibana
template: src=monit/kibana.monit dest={{ monit_config_home }}/kibana.monit
when: ("web" in group_names) and (install_elasticsearch | default(True))
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/monit/tasks/scripts.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/scripts.yml b/metron-deployment/roles/monit/tasks/scripts.yml
index df1a443..43901d2 100644
--- a/metron-deployment/roles/monit/tasks/scripts.yml
+++ b/metron-deployment/roles/monit/tasks/scripts.yml
@@ -39,6 +39,7 @@
- bro
- pcap
- enrichment
+ - indexing
- name: Deploy topology status scripts
template:
@@ -51,3 +52,4 @@
- bro
- pcap
- enrichment
+ - indexing
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/monit/templates/monit/enrichment-elasticsearch.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/enrichment-elasticsearch.monit b/metron-deployment/roles/monit/templates/monit/enrichment-elasticsearch.monit
deleted file mode 100644
index ceae475..0000000
--- a/metron-deployment/roles/monit/templates/monit/enrichment-elasticsearch.monit
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-check program enrichment with path "{{ monit_home }}/status_enrichment_topology.sh"
- start program "{{ metron_directory }}/bin/start_elasticsearch_topology.sh"
- stop program "{{ monit_home }}/stop_enrichment_topology.sh"
- if status != 0 then restart
- group yaf
- group bro
- group snort
- group enrichments
- group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/monit/templates/monit/enrichment-solr.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/enrichment-solr.monit b/metron-deployment/roles/monit/templates/monit/enrichment-solr.monit
deleted file mode 100644
index 94f4858..0000000
--- a/metron-deployment/roles/monit/templates/monit/enrichment-solr.monit
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-check program enrichment with path "{{ monit_home }}/status_enrichment_topology.sh"
- start program "{{ metron_directory }}/bin/start_solr_topology.sh"
- stop program "{{ monit_home }}/stop_enrichment_topology.sh"
- if status != 0 then restart
- group yaf
- group bro
- group snort
- group enrichments
- group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/monit/templates/monit/enrichment.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/enrichment.monit b/metron-deployment/roles/monit/templates/monit/enrichment.monit
new file mode 100644
index 0000000..43d197e
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/enrichment.monit
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check program enrichment with path "{{ monit_home }}/status_enrichment_topology.sh"
+ start program "{{ metron_directory }}/bin/start_enrichment_topology.sh"
+ stop program "{{ monit_home }}/stop_enrichment_topology.sh"
+ if status != 0 then restart
+ group yaf
+ group bro
+ group snort
+ group enrichments
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/monit/templates/monit/indexing-elasticsearch.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/indexing-elasticsearch.monit b/metron-deployment/roles/monit/templates/monit/indexing-elasticsearch.monit
new file mode 100644
index 0000000..86992e4
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/indexing-elasticsearch.monit
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check program indexing with path "{{ monit_home }}/status_indexing_topology.sh"
+ start program "{{ metron_directory }}/bin/start_elasticsearch_topology.sh"
+ stop program "{{ monit_home }}/stop_indexing_topology.sh"
+ if status != 0 then restart
+ group yaf
+ group bro
+ group snort
+ group enrichments
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-deployment/roles/monit/templates/monit/indexing-solr.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/indexing-solr.monit b/metron-deployment/roles/monit/templates/monit/indexing-solr.monit
new file mode 100644
index 0000000..432a0e6
--- /dev/null
+++ b/metron-deployment/roles/monit/templates/monit/indexing-solr.monit
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+check program indexing with path "{{ monit_home }}/status_indexing_topology.sh"
+ start program "{{ metron_directory }}/bin/start_solr_topology.sh"
+ stop program "{{ monit_home }}/stop_indexing_topology.sh"
+ if status != 0 then restart
+ group yaf
+ group bro
+ group snort
+ group enrichments
+ group metron
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index 30162dd..d625402 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -294,6 +294,10 @@
<pattern>com.google.common</pattern>
<shadedPattern>org.apache.metron.guava</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>org.apache.commons.beanutils</pattern>
+ <shadedPattern>org.apache.metron.beanutils</shadedPattern>
+ </relocation>
</relocations>
<transformers>
<transformer
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
index 1175e8b..d1f5748 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
@@ -27,6 +27,8 @@ public class Constants {
public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000;
public static final String SENSOR_TYPE = "source.type";
public static final String ENRICHMENT_TOPIC = "enrichments";
+ public static final String INDEXING_TOPIC = "indexing";
+ public static final String INDEXING_ERROR_TOPIC = "indexing_error";
public static final String DEFAULT_PARSER_ERROR_TOPIC = "parser_error";
public static final String DEFAULT_PARSER_INVALID_TOPIC = "parser_invalid";
public static final String ERROR_STREAM = "error";
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/BloomFilter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/BloomFilter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/BloomFilter.java
new file mode 100644
index 0000000..82172f5
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/BloomFilter.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.utils;
+
+import com.google.common.hash.Funnel;
+import com.google.common.hash.PrimitiveSink;
+
+import java.io.Serializable;
+import java.util.function.Function;
+
+public class BloomFilter<T> implements Serializable {
+
+ private static class BloomFunnel<T> implements Funnel<T>, Serializable {
+ Function<T, byte[]> serializer;
+ public BloomFunnel(Function<T, byte[]> serializer) {
+ this.serializer = serializer;
+ }
+ @Override
+ public void funnel(T obj, PrimitiveSink primitiveSink) {
+ primitiveSink.putBytes(serializer.apply(obj));
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return this.getClass().equals(obj.getClass());
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode() * 31;
+ }
+ }
+ private com.google.common.hash.BloomFilter<T> filter;
+ public BloomFilter(Function<T, byte[]> serializer, int expectedInsertions, double falsePositiveRate) {
+ filter = com.google.common.hash.BloomFilter.create(new BloomFunnel<T>(serializer), expectedInsertions, falsePositiveRate);
+ }
+
+ public boolean mightContain(T key) {
+ return filter.mightContain(key);
+ }
+ public void add(T key) {
+ filter.put(key);
+ }
+ public void merge(BloomFilter<T> filter2) {
+ filter.putAll(filter2.filter);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
new file mode 100644
index 0000000..6d2af72
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.utils;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public enum KafkaUtils {
+ INSTANCE;
+ public List<String> getBrokersFromZookeeper(String zkQuorum) throws Exception {
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ CuratorFramework framework = CuratorFrameworkFactory.newClient(zkQuorum, retryPolicy);
+ framework.start();
+ try {
+ return getBrokersFromZookeeper(framework);
+ } finally {
+ framework.close();
+ }
+ }
+ public List<String> getBrokersFromZookeeper(CuratorFramework client) throws Exception {
+ List<String> ret = new ArrayList<>();
+ for(String id : client.getChildren().forPath("/brokers/ids")) {
+ byte[] data = client.getData().forPath("/brokers/ids/" + id);
+ String brokerInfoStr = new String(data);
+ Map<String, Object> brokerInfo = JSONUtils.INSTANCE.load(brokerInfoStr, new TypeReference<Map<String, Object>>() {
+ });
+ ret.add(brokerInfo.get("host") + ":" + brokerInfo.get("port"));
+ }
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/AbstractWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/AbstractWriter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/AbstractWriter.java
deleted file mode 100644
index 56a4e48..0000000
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/AbstractWriter.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.common.writer;
-
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-
-public abstract class AbstractWriter {
- public AbstractWriter() {}
- public abstract void configure(String sensorName, WriterConfiguration configuration);
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterComponent.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterComponent.java
deleted file mode 100644
index c0e4f37..0000000
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/BulkWriterComponent.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.common.writer;
-
-import backtype.storm.task.OutputCollector;
-import backtype.storm.tuple.Tuple;
-import com.google.common.collect.Iterables;
-import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.interfaces.BulkMessageWriter;
-import org.apache.metron.common.utils.ErrorUtils;
-import org.apache.metron.common.utils.MessageUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-import java.util.function.Function;
-
-public class BulkWriterComponent<MESSAGE_T> {
- public static final Logger LOG = LoggerFactory
- .getLogger(BulkWriterComponent.class);
- private Map<String, Collection<Tuple>> sensorTupleMap = new HashMap<>();
- private Map<String, List<MESSAGE_T>> sensorMessageMap = new HashMap<>();
- private OutputCollector collector;
- private boolean handleCommit = true;
- private boolean handleError = true;
- public BulkWriterComponent(OutputCollector collector) {
- this.collector = collector;
- }
-
- public BulkWriterComponent(OutputCollector collector, boolean handleCommit, boolean handleError) {
- this(collector);
- this.handleCommit = handleCommit;
- this.handleError = handleError;
- }
-
- public void commit(Iterable<Tuple> tuples) {
- tuples.forEach(t -> collector.ack(t));
- if(LOG.isDebugEnabled()) {
- LOG.debug("Acking " + Iterables.size(tuples) + " tuples");
- }
- }
-
- public void error(Throwable e, Iterable<Tuple> tuples) {
- tuples.forEach(t -> collector.ack(t));
- if(!Iterables.isEmpty(tuples)) {
- LOG.error("Failing " + Iterables.size(tuples) + " tuples", e);
- ErrorUtils.handleError(collector, e, Constants.ERROR_STREAM);
- }
- }
-
- protected Collection<Tuple> createTupleCollection() {
- return new ArrayList<>();
- }
-
-
- public void errorAll(Throwable e) {
- for(Map.Entry<String, Collection<Tuple>> kv : sensorTupleMap.entrySet()) {
- error(e, kv.getValue());
- sensorTupleMap.remove(kv.getKey());
- sensorMessageMap.remove(kv.getKey());
- }
- }
-
- public void errorAll(String sensorType, Throwable e) {
- error(e, Optional.ofNullable(sensorTupleMap.get(sensorType)).orElse(new ArrayList<>()));
- sensorTupleMap.remove(sensorType);
- sensorMessageMap.remove(sensorType);
- }
- public void write( String sensorType
- , Tuple tuple
- , MESSAGE_T message
- , BulkMessageWriter<MESSAGE_T> bulkMessageWriter
- , WriterConfiguration configurations
- ) throws Exception
- {
- int batchSize = configurations.getBatchSize(sensorType);
- Collection<Tuple> tupleList = sensorTupleMap.get(sensorType);
- if (tupleList == null) {
- tupleList = createTupleCollection();
- }
- tupleList.add(tuple);
- List<MESSAGE_T> messageList = sensorMessageMap.get(sensorType);
- if (messageList == null) {
- messageList = new ArrayList<>();
- }
- messageList.add(message);
-
- if (tupleList.size() < batchSize) {
- sensorTupleMap.put(sensorType, tupleList);
- sensorMessageMap.put(sensorType, messageList);
- } else {
- try {
- bulkMessageWriter.write(sensorType, configurations, tupleList, messageList);
- if(handleCommit) {
- commit(tupleList);
- }
-
- } catch (Throwable e) {
- if(handleError) {
- error(e, tupleList);
- }
- else {
- throw e;
- }
- }
- finally {
- sensorTupleMap.remove(sensorType);
- sensorMessageMap.remove(sensorType);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/NoopWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/NoopWriter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/NoopWriter.java
deleted file mode 100644
index 3b27f75..0000000
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/NoopWriter.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.common.writer;
-
-import backtype.storm.tuple.Tuple;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Iterables;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.interfaces.BulkMessageWriter;
-import org.apache.metron.common.utils.ConversionUtils;
-
-import java.io.Closeable;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.function.Function;
-
-public class NoopWriter extends AbstractWriter implements BulkMessageWriter<Object> {
-
- public static class RandomLatency implements Function<Void, Void> {
- private int min;
- private int max;
-
- public RandomLatency(int min, int max) {
- this.min = min;
- this.max = max;
- }
-
- public int getMin() {
- return min;
- }
- public int getMax() {
- return max;
- }
-
- @Override
- public Void apply(Void aVoid) {
- int sleepMs = ThreadLocalRandom.current().nextInt(min, max + 1);
- try {
- Thread.sleep(sleepMs);
- } catch (InterruptedException e) {
- }
- return null;
- }
- }
-
- public static class FixedLatency implements Function<Void, Void> {
- private int latency;
- public FixedLatency(int latency) {
- this.latency = latency;
- }
- public int getLatency() {
- return latency;
- }
-
- @Override
- public Void apply(Void aVoid) {
- if(latency > 0) {
- try {
- Thread.sleep(latency);
- } catch (InterruptedException e) {
- }
- }
- return null;
- }
- }
- Function<Void, Void> sleepFunction = null;
-
- public NoopWriter withLatency(String sleepConfig) {
- sleepFunction = getSleepFunction(sleepConfig);
- return this;
- }
-
-
- private Function<Void, Void> getSleepFunction(String sleepConfig) {
- String usageMessage = "Unexpected: " + sleepConfig + " Expected value: integer for a fixed sleep duration in milliseconds (e.g. 10) " +
- "or a range of latencies separated by a comma (e.g. \"10, 20\") to sleep a random amount in that range.";
- try {
- if (sleepConfig.contains(",")) {
- // random latency within a range.
- Iterable<String> it = Splitter.on(',').split(sleepConfig);
- Integer min = ConversionUtils.convert(Iterables.getFirst(it, "").trim(), Integer.class);
- Integer max= ConversionUtils.convert(Iterables.getLast(it, "").trim(), Integer.class);
- if (min != null && max != null) {
- return new RandomLatency(min, max);
- }
- } else {
- //fixed latency
- Integer latency = ConversionUtils.convert(sleepConfig.trim(), Integer.class);
- if(latency != null) {
- return new FixedLatency(latency);
- }
- }
- }
- catch(Throwable t) {
- throw new IllegalArgumentException(usageMessage, t);
- }
- throw new IllegalArgumentException(usageMessage);
- }
-
- @Override
- public void configure(String sensorName, WriterConfiguration configuration) {
- Map<String, Object> config = configuration.getSensorConfig(sensorName);
- if(config != null) {
- Object noopLatency = config.get("noopLatency");
- if(noopLatency != null) {
- sleepFunction = getSleepFunction(noopLatency.toString());
- }
- }
- }
-
- @Override
- public void init(Map stormConf, WriterConfiguration config) throws Exception {
- }
-
- @Override
- public void write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<Object> messages) throws Exception {
- if(sleepFunction != null) {
- sleepFunction.apply(null);
- }
- }
-
- @Override
- public void close() throws Exception {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/WriterToBulkWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/WriterToBulkWriter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/WriterToBulkWriter.java
deleted file mode 100644
index b0bde6c..0000000
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/writer/WriterToBulkWriter.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.common.writer;
-
-import backtype.storm.tuple.Tuple;
-import com.google.common.collect.Iterables;
-import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFacade;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.interfaces.BulkMessageWriter;
-import org.apache.metron.common.interfaces.MessageWriter;
-
-import java.util.List;
-import java.util.Map;
-
-public class WriterToBulkWriter<MESSAGE_T> implements BulkMessageWriter<MESSAGE_T> {
- MessageWriter<MESSAGE_T> messageWriter;
-
- public WriterToBulkWriter(MessageWriter<MESSAGE_T> messageWriter) {
- this.messageWriter = messageWriter;
- }
- @Override
- public void init(Map stormConf, WriterConfiguration config) throws Exception {
- messageWriter.init();
- }
-
- @Override
- public void write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<MESSAGE_T> messages) throws Exception {
- if(messages.size() > 1) {
- throw new IllegalStateException("WriterToBulkWriter expects a batch of exactly 1");
- }
- messageWriter.write(sensorType, configurations, Iterables.getFirst(tuples, null), Iterables.getFirst(messages, null));
- }
-
- @Override
- public void close() throws Exception {
- messageWriter.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-common/src/test/java/org/apache/metron/common/writer/NoopWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/writer/NoopWriterTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/writer/NoopWriterTest.java
deleted file mode 100644
index e40ce7c..0000000
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/writer/NoopWriterTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.common.writer;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-public class NoopWriterTest {
- @Test
- public void testFixedLatencyConfig() {
- NoopWriter writer = new NoopWriter().withLatency("10");
- Assert.assertTrue(writer.sleepFunction instanceof NoopWriter.FixedLatency);
- NoopWriter.FixedLatency sleepFunction = (NoopWriter.FixedLatency)writer.sleepFunction;
- Assert.assertEquals(10, sleepFunction.getLatency());
- }
-
- private void ensureRandomLatencyConfig(String latencyConfig, int min, int max) {
- NoopWriter writer = new NoopWriter().withLatency(latencyConfig);
- Assert.assertTrue(writer.sleepFunction instanceof NoopWriter.RandomLatency);
- NoopWriter.RandomLatency sleepFunction = (NoopWriter.RandomLatency)writer.sleepFunction;
- Assert.assertEquals(min, sleepFunction.getMin());
- Assert.assertEquals(max, sleepFunction.getMax());
- }
-
- @Test
- public void testRandomLatencyConfig() {
- ensureRandomLatencyConfig("10,20", 10, 20);
- ensureRandomLatencyConfig("10, 20", 10, 20);
- ensureRandomLatencyConfig("10 ,20", 10, 20);
- ensureRandomLatencyConfig("10 , 20", 10, 20);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index 4db3c07..3ecd84d 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -40,6 +40,16 @@
<groupId>org.apache.metron</groupId>
<artifactId>metron-enrichment</artifactId>
<version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
@@ -132,6 +142,25 @@
<version>${global_mockito_version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-indexing</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-indexing</artifactId>
+ <version>${project.parent.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-enrichment</artifactId>
+ <version>${project.parent.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<reporting>
<plugins>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
index e2370ab..a1516d6 100644
--- a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
+++ b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
@@ -14,22 +14,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+##### Storm #####
+indexing.workers=1
+indexing.executors=0
##### Kafka #####
kafka.zk=node1:2181
kafka.broker=node1:6667
-spout.kafka.topic.asa=asa
-spout.kafka.topic.bro=bro
-spout.kafka.topic.fireeye=fireeye
-spout.kafka.topic.ise=ise
-spout.kafka.topic.lancope=lancope
-spout.kafka.topic.paloalto=paloalto
-spout.kafka.topic.pcap=pcap
-spout.kafka.topic.snort=snort
-spout.kafka.topic.yaf=yaf
+kafka.start=WHERE_I_LEFT_OFF
##### Indexing #####
+index.input.topic=indexing
+index.error.topic=indexing_error
writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter
##### ElasticSearch #####
@@ -38,13 +35,6 @@ es.ip=10.22.0.214
es.port=9300
es.clustername=elasticsearch
-##### MySQL #####
-
-mysql.ip=10.22.0.214
-mysql.port=3306
-mysql.username=root
-mysql.password=hadoop123
-
##### Metrics #####
#reporters
@@ -63,23 +53,6 @@ org.apache.metron.metrics.TelemetryParserBolt.emits=true
org.apache.metron.metrics.TelemetryParserBolt.fails=true
-#GenericEnrichmentBolt
-org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
-org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
-org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
-
-
-#TelemetryIndexingBolt
-org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
-org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
-org.apache.metron.metrics.TelemetryIndexingBolt.fails=true
-
-##### Host Enrichment #####
-
-org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
-{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
-{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
-
##### HDFS #####
bolt.hdfs.batch.size=5000
@@ -91,19 +64,3 @@ bolt.hdfs.finished.file.path=/paloalto/rotated
bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
index.hdfs.output=/tmp/metron/enriched
-##### HBase #####
-bolt.hbase.table.name=pcap
-bolt.hbase.table.fields=t:value
-bolt.hbase.table.key.tuple.field.name=key
-bolt.hbase.table.timestamp.tuple.field.name=timestamp
-bolt.hbase.enable.batching=false
-bolt.hbase.write.buffer.size.in.bytes=2000000
-bolt.hbase.durability=SKIP_WAL
-bolt.hbase.partitioner.region.info.refresh.interval.mins=60
-
-##### Threat Intel #####
-
-threat.intel.tracker.table=
-threat.intel.tracker.cf=
-threat.intel.ip.table=
-threat.intel.ip.cf=
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh b/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh
index 9149d09..16f83d3 100755
--- a/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh
+++ b/metron-platform/metron-elasticsearch/src/main/scripts/start_elasticsearch_topology.sh
@@ -19,4 +19,4 @@
METRON_VERSION=${project.version}
METRON_HOME=/usr/metron/$METRON_VERSION
TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION.jar
-storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/enrichment/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties
+storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/remote.yaml --filter $METRON_HOME/config/elasticsearch.properties
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java
deleted file mode 100644
index e0bbb89..0000000
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.elasticsearch.integration;
-
-import org.apache.metron.TestConstants;
-import org.apache.metron.common.interfaces.FieldNameConverter;
-import org.apache.metron.elasticsearch.writer.ElasticsearchFieldNameConverter;
-import org.apache.metron.integration.EnrichmentIntegrationTest;
-import org.apache.metron.integration.ComponentRunner;
-import org.apache.metron.integration.InMemoryComponent;
-import org.apache.metron.integration.Processor;
-import org.apache.metron.integration.ReadinessState;
-import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-public class ElasticsearchEnrichmentIntegrationTest extends EnrichmentIntegrationTest {
-
- private String indexDir = "target/elasticsearch";
- private String dateFormat = "yyyy.MM.dd.HH";
- private String index = "yaf_index_" + new SimpleDateFormat(dateFormat).format(new Date());
- private FieldNameConverter fieldNameConverter = new ElasticsearchFieldNameConverter();
-
- @Override
- public FieldNameConverter getFieldNameConverter() {
- return fieldNameConverter;
- }
-
- @Override
- public InMemoryComponent getSearchComponent(final Properties topologyProperties) {
- return new ElasticSearchComponent.Builder()
- .withHttpPort(9211)
- .withIndexDir(new File(indexDir))
- .build();
- }
-
- @Override
- public Processor<List<Map<String, Object>>> getProcessor(final List<byte[]> inputMessages) {
- return new Processor<List<Map<String, Object>>>() {
- List<Map<String, Object>> docs = null;
- public ReadinessState process(ComponentRunner runner) {
- ElasticSearchComponent elasticSearchComponent = runner.getComponent("search", ElasticSearchComponent.class);
- if (elasticSearchComponent.hasIndex(index)) {
- List<Map<String, Object>> docsFromDisk;
- try {
- docs = elasticSearchComponent.getAllIndexedDocs(index, testSensorType + "_doc");
- docsFromDisk = readDocsFromDisk(hdfsDir);
- System.out.println(docs.size() + " vs " + inputMessages.size() + " vs " + docsFromDisk.size());
- } catch (IOException e) {
- throw new IllegalStateException("Unable to retrieve indexed documents.", e);
- }
- if (docs.size() < inputMessages.size() || docs.size() != docsFromDisk.size()) {
- return ReadinessState.NOT_READY;
- } else {
- return ReadinessState.READY;
- }
- } else {
- return ReadinessState.NOT_READY;
- }
- }
-
- public List<Map<String, Object>> getResult() {
- return docs;
- }
- };
- }
-
- @Override
- public void setAdditionalProperties(Properties topologyProperties) {
- topologyProperties.setProperty("writer.class.name", "org.apache.metron.elasticsearch.writer.ElasticsearchWriter");
- }
-
- @Override
- public String cleanField(String field) {
- return field;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java
new file mode 100644
index 0000000..b27c545
--- /dev/null
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchIndexingIntegrationTest.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.elasticsearch.integration;
+
+import org.apache.metron.common.interfaces.FieldNameConverter;
+import org.apache.metron.elasticsearch.writer.ElasticsearchFieldNameConverter;
+import org.apache.metron.indexing.integration.IndexingIntegrationTest;
+import org.apache.metron.integration.ComponentRunner;
+import org.apache.metron.integration.InMemoryComponent;
+import org.apache.metron.integration.Processor;
+import org.apache.metron.integration.ReadinessState;
+import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class ElasticsearchIndexingIntegrationTest extends IndexingIntegrationTest {
+
+ private String indexDir = "target/elasticsearch";
+ private String dateFormat = "yyyy.MM.dd.HH";
+ private String index = "yaf_index_" + new SimpleDateFormat(dateFormat).format(new Date());
+ private FieldNameConverter fieldNameConverter = new ElasticsearchFieldNameConverter();
+
+ @Override
+ public FieldNameConverter getFieldNameConverter() {
+ return fieldNameConverter;
+ }
+
+ @Override
+ public InMemoryComponent getSearchComponent(final Properties topologyProperties) {
+ return new ElasticSearchComponent.Builder()
+ .withHttpPort(9211)
+ .withIndexDir(new File(indexDir))
+ .build();
+ }
+
+ @Override
+ public Processor<List<Map<String, Object>>> getProcessor(final List<byte[]> inputMessages) {
+ return new Processor<List<Map<String, Object>>>() {
+ List<Map<String, Object>> docs = null;
+ public ReadinessState process(ComponentRunner runner) {
+ ElasticSearchComponent elasticSearchComponent = runner.getComponent("search", ElasticSearchComponent.class);
+ if (elasticSearchComponent.hasIndex(index)) {
+ List<Map<String, Object>> docsFromDisk;
+ try {
+ docs = elasticSearchComponent.getAllIndexedDocs(index, testSensorType + "_doc");
+ docsFromDisk = readDocsFromDisk(hdfsDir);
+ System.out.println(docs.size() + " vs " + inputMessages.size() + " vs " + docsFromDisk.size());
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to retrieve indexed documents.", e);
+ }
+ if (docs.size() < inputMessages.size() || docs.size() != docsFromDisk.size()) {
+ return ReadinessState.NOT_READY;
+ } else {
+ return ReadinessState.READY;
+ }
+ } else {
+ return ReadinessState.NOT_READY;
+ }
+ }
+
+ public List<Map<String, Object>> getResult() {
+ return docs;
+ }
+ };
+ }
+
+ @Override
+ public void setAdditionalProperties(Properties topologyProperties) {
+ topologyProperties.setProperty("es.clustername", "metron");
+ topologyProperties.setProperty("es.port", "9300");
+ topologyProperties.setProperty("es.ip", "localhost");
+ topologyProperties.setProperty("writer.class.name", "org.apache.metron.elasticsearch.writer.ElasticsearchWriter");
+ }
+
+ @Override
+ public String cleanField(String field) {
+ return field;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml
index 54c8fd5..84891e8 100644
--- a/metron-platform/metron-enrichment/pom.xml
+++ b/metron-platform/metron-enrichment/pom.xml
@@ -37,10 +37,25 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
+ <artifactId>metron-writer</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
<artifactId>metron-hbase</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${global_jackson_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${global_jackson_version}</version>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
@@ -65,7 +80,6 @@
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -77,7 +91,6 @@
<groupId>javax.servlet</groupId>
</exclusion>
</exclusions>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
@@ -95,36 +108,36 @@
</exclusion>
</exclusions>
</dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${global_hbase_guava_version}</version>
+ </dependency>
<dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-hdfs</artifactId>
- <version>${global_storm_version}</version>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.9.2</artifactId>
+ <version>${global_kafka_version}</version>
<exclusions>
<exclusion>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${global_guava_version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${global_hadoop_version}</version>
- <scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>commons-httpclient</artifactId>
+ <groupId>commons-httpclient</groupId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -144,9 +157,16 @@
<version>${project.parent.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-integration-test</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<reporting>
<plugins>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
@@ -170,6 +190,7 @@
<dependencyLocationsEnabled>false</dependencyLocationsEnabled>
</configuration>
</plugin>
+
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>emma-maven-plugin</artifactId>
@@ -188,6 +209,17 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
@@ -214,10 +246,12 @@
<goal>shade</goal>
</goals>
<configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
<relocations>
<relocation>
- <pattern>com.google.common</pattern>
- <shadedPattern>org.apache.metron.guava</shadedPattern>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>org.apache.metron.jackson</shadedPattern>
</relocation>
</relocations>
<transformers>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
index 6c9165c..4358370 100644
--- a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
@@ -53,5 +53,13 @@
<fileMode>0644</fileMode>
<lineEnding>unix</lineEnding>
</fileSet>
+ <fileSet>
+ <directory>${project.basedir}/target</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-uber.jar</include>
+ </includes>
+ <outputDirectory>/lib</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ </fileSet>
</fileSets>
</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/config/enrichment.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment.properties b/metron-platform/metron-enrichment/src/main/config/enrichment.properties
new file mode 100644
index 0000000..e32915c
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/config/enrichment.properties
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+##### Kafka #####
+
+kafka.zk=node1:2181
+kafka.broker=node1:6667
+enrichment.output.topic=indexing
+
+##### MySQL #####
+
+mysql.ip=10.22.0.214
+mysql.port=3306
+mysql.username=root
+mysql.password=hadoop123
+
+##### Metrics #####
+
+#reporters
+org.apache.metron.metrics.reporter.graphite=true
+org.apache.metron.metrics.reporter.console=false
+org.apache.metron.metrics.reporter.jmx=false
+
+#Graphite Addresses
+
+org.apache.metron.metrics.graphite.address=localhost
+org.apache.metron.metrics.graphite.port=2023
+
+#TelemetryParserBolt
+org.apache.metron.metrics.TelemetryParserBolt.acks=true
+org.apache.metron.metrics.TelemetryParserBolt.emits=true
+org.apache.metron.metrics.TelemetryParserBolt.fails=true
+
+
+#GenericEnrichmentBolt
+org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
+org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
+org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
+
+
+#TelemetryIndexingBolt
+org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
+org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
+org.apache.metron.metrics.TelemetryIndexingBolt.fails=true
+
+##### Host Enrichment #####
+
+org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
+{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
+{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
+
+
+##### HBase #####
+bolt.hbase.table.name=pcap
+bolt.hbase.table.fields=t:value
+bolt.hbase.table.key.tuple.field.name=key
+bolt.hbase.table.timestamp.tuple.field.name=timestamp
+bolt.hbase.enable.batching=false
+bolt.hbase.write.buffer.size.in.bytes=2000000
+bolt.hbase.durability=SKIP_WAL
+bolt.hbase.partitioner.region.info.refresh.interval.mins=60
+
+##### Threat Intel #####
+
+threat.intel.tracker.table=
+threat.intel.tracker.cf=
+threat.intel.ip.table=
+threat.intel.ip.cf=
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
index b499b24..69d5f66 100644
--- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
+++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
@@ -129,37 +129,16 @@ components:
args:
- ref: "simpleHBaseThreatIntelEnrichment"
- - id: "fileNameFormat"
- className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat"
- configMethods:
- - name: "withPrefix"
- args:
- - "enrichment-"
- - name: "withExtension"
- args:
- - ".json"
- - name: "withPath"
- args:
- - "${index.hdfs.output}"
-
- - id: "hdfsRotationPolicy"
- className: "${bolt.hdfs.rotation.policy}"
- constructorArgs:
- - ${bolt.hdfs.rotation.policy.count}
- - "${bolt.hdfs.rotation.policy.units}"
#indexing
- - id: "hdfsWriter"
- className: "org.apache.metron.writer.hdfs.HdfsWriter"
+ - id: "kafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
configMethods:
- - name: "withFileNameFormat"
+ - name: "withTopic"
args:
- - ref: "fileNameFormat"
- - name: "withRotationPolicy"
+ - "${enrichment.output.topic}"
+ - name: "withZkQuorum"
args:
- - ref: "hdfsRotationPolicy"
-
- - id: "indexWriter"
- className: "${writer.class.name}"
+ - "${kafka.zk}"
#kafka/zookeeper
- id: "zkHosts"
@@ -277,22 +256,15 @@ bolts:
- name: "withMaxTimeRetain"
args: [10]
# Indexing Bolts
- - id: "indexingBolt"
- className: "org.apache.metron.enrichment.bolt.BulkMessageWriterBolt"
- constructorArgs:
- - "${kafka.zk}"
- configMethods:
- - name: "withBulkMessageWriter"
- args:
- - ref: "indexWriter"
- - id: "hdfsIndexingBolt"
- className: "org.apache.metron.enrichment.bolt.BulkMessageWriterBolt"
+
+ - id: "outputBolt"
+ className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
constructorArgs:
- "${kafka.zk}"
configMethods:
- - name: "withBulkMessageWriter"
+ - name: "withMessageWriter"
args:
- - ref: "hdfsWriter"
+ - ref: "kafkaWriter"
streams:
@@ -389,25 +361,12 @@ streams:
streamId: "message"
type: FIELDS
args: ["key"]
-#indexing
- - name: "threatIntelJoin -> indexing"
+#output
+ - name: "threatIntelJoin -> output"
from: "threatIntelJoinBolt"
- to: "indexingBolt"
+ to: "outputBolt"
grouping:
streamId: "message"
type: FIELDS
args: ["key"]
- - name: "threatIntelJoin -> hdfs"
- from: "threatIntelJoinBolt"
- to: "hdfsIndexingBolt"
- grouping:
- streamId: "message"
- type: SHUFFLE
-
- - name: "indexingBolt -> errorIndexingBolt"
- from: "indexingBolt"
- to: "indexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
[3/5] incubator-metron git commit: METRON-154: Decouple enrichment
and indexing closes apache/incubator-metron#192
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-indexing/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml
new file mode 100644
index 0000000..9760a11
--- /dev/null
+++ b/metron-platform/metron-indexing/pom.xml
@@ -0,0 +1,253 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-platform</artifactId>
+ <version>0.2.0BETA</version>
+ </parent>
+ <artifactId>metron-indexing</artifactId>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ <slf4j.version>1.7.7</slf4j.version>
+ <storm.hdfs.version>0.1.2</storm.hdfs.version>
+ <guava.version>${global_hbase_guava_version}</guava.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${global_hadoop_version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ </exclusions>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${global_storm_version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-hdfs</artifactId>
+ <version>${global_storm_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${global_guava_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${global_hadoop_version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${global_mockito_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-validator</groupId>
+ <artifactId>commons-validator</artifactId>
+ <version>1.4.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-test-utilities</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-integration-test</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-enrichment</artifactId>
+ <version>${project.parent.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <systemProperties>
+ <property>
+ <name>mode</name>
+ <value>global</value>
+ </property>
+ </systemProperties>
+ </configuration>
+ </plugin>
+
+ <!-- Normally, dependency report takes time, skip it -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>2.7</version>
+
+ <configuration>
+ <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>emma-maven-plugin</artifactId>
+ <version>1.0-alpha-3</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-pmd-plugin</artifactId>
+ <configuration>
+ <targetJdk>${global_java_version}</targetJdk>
+ </configuration>
+ </plugin>
+ </plugins>
+ </reporting>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <source>${global_java_version}</source>
+ <target>${global_java_version}</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${global_shade_version}</version>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <artifactSet>
+ <excludes>
+ <exclude>*slf4j*</exclude>
+ </excludes>
+ </artifactSet>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>org.apache.metron.guava</shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resource>.yaml</resource>
+ </transformer>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass></mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-indexing/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/assembly/assembly.xml b/metron-platform/metron-indexing/src/main/assembly/assembly.xml
new file mode 100644
index 0000000..6c9165c
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/assembly/assembly.xml
@@ -0,0 +1,57 @@
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<assembly>
+ <id>archive</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}/src/main/config</directory>
+ <outputDirectory>/config</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0644</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/src/main/scripts</directory>
+ <outputDirectory>/bin</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/src/main/flux</directory>
+ <outputDirectory>/flux</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0644</fileMode>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ </fileSets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml b/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml
new file mode 100644
index 0000000..d1e697b
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/flux/indexing/remote.yaml
@@ -0,0 +1,152 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "indexing"
+config:
+ topology.workers: ${indexing.workers}
+ topology.acker.executors: ${indexing.executors}
+
+components:
+
+ - id: "fileNameFormat"
+ className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat"
+ configMethods:
+ - name: "withPrefix"
+ args:
+ - "enrichment-"
+ - name: "withExtension"
+ args:
+ - ".json"
+ - name: "withPath"
+ args:
+ - "${index.hdfs.output}"
+
+ - id: "hdfsRotationPolicy"
+ className: "${bolt.hdfs.rotation.policy}"
+ constructorArgs:
+ - ${bolt.hdfs.rotation.policy.count}
+ - "${bolt.hdfs.rotation.policy.units}"
+#indexing
+ - id: "hdfsWriter"
+ className: "org.apache.metron.writer.hdfs.HdfsWriter"
+ configMethods:
+ - name: "withFileNameFormat"
+ args:
+ - ref: "fileNameFormat"
+ - name: "withRotationPolicy"
+ args:
+ - ref: "hdfsRotationPolicy"
+ - id: "kafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
+ configMethods:
+ - name: "withTopic"
+ args:
+ - "${index.error.topic}"
+ - name: "withZkQuorum"
+ args:
+ - "${kafka.zk}"
+
+ - id: "indexWriter"
+ className: "${writer.class.name}"
+
+#kafka/zookeeper
+ - id: "zkHosts"
+ className: "storm.kafka.ZkHosts"
+ constructorArgs:
+ - "${kafka.zk}"
+ - id: "kafkaConfig"
+ className: "org.apache.metron.common.spout.kafka.SpoutConfig"
+ constructorArgs:
+ # zookeeper hosts
+ - ref: "zkHosts"
+ # topic name
+ - "${index.input.topic}"
+ # zk root
+ - ""
+ # id
+ - "indexing"
+ configMethods:
+ - name: "from"
+ args:
+ - "${kafka.start}"
+
+spouts:
+ - id: "kafkaSpout"
+ className: "storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+bolts:
+
+# Indexing Bolts
+ - id: "indexingBolt"
+ className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withBulkMessageWriter"
+ args:
+ - ref: "indexWriter"
+ - name: "withMessageGetter"
+ args:
+ - "RAW"
+ - id: "hdfsIndexingBolt"
+ className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withBulkMessageWriter"
+ args:
+ - ref: "hdfsWriter"
+ - name: "withMessageGetter"
+ args:
+ - "RAW"
+
+ - id: "indexingErrorBolt"
+ className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ configMethods:
+ - name: "withMessageWriter"
+ args:
+ - ref: "kafkaWriter"
+
+streams:
+
+ - name: "spout -> indexing"
+ from: "kafkaSpout"
+ to: "indexingBolt"
+ grouping:
+ type: SHUFFLE
+
+ - name: "spout -> hdfs"
+ from: "kafkaSpout"
+ to: "hdfsIndexingBolt"
+ grouping:
+ type: SHUFFLE
+
+ - name: "indexingBolt -> errorIndexingBolt"
+ from: "indexingBolt"
+ to: "indexingErrorBolt"
+ grouping:
+ streamId: "error"
+ type: SHUFFLE
+
+ - name: "hdfsBolt -> errorIndexingBolt"
+ from: "hdfsIndexingBolt"
+ to: "indexingErrorBolt"
+ grouping:
+ streamId: "error"
+ type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
new file mode 100644
index 0000000..2c85308
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
@@ -0,0 +1,240 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.integration;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import org.apache.metron.TestConstants;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.interfaces.FieldNameConverter;
+import org.apache.metron.common.spout.kafka.SpoutConfig;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
+import org.apache.metron.integration.BaseIntegrationTest;
+import org.apache.metron.integration.ComponentRunner;
+import org.apache.metron.integration.InMemoryComponent;
+import org.apache.metron.integration.Processor;
+import org.apache.metron.integration.components.FluxTopologyComponent;
+import org.apache.metron.integration.components.KafkaWithZKComponent;
+import org.apache.metron.integration.utils.TestUtils;
+import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy;
+import org.junit.Assert;
+import org.junit.Test;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+
+public abstract class IndexingIntegrationTest extends BaseIntegrationTest {
+ protected String hdfsDir = "target/indexingIntegrationTest/hdfs";
+ protected String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
+ protected String fluxPath = "../metron-indexing/src/main/flux/indexing/remote.yaml";
+ protected String testSensorType = "test";
+
+
+ public static List<Map<String, Object>> readDocsFromDisk(String hdfsDirStr) throws IOException {
+ List<Map<String, Object>> ret = new ArrayList<>();
+ File hdfsDir = new File(hdfsDirStr);
+ Stack<File> fs = new Stack<>();
+ if (hdfsDir.exists()) {
+ fs.push(hdfsDir);
+ while (!fs.empty()) {
+ File f = fs.pop();
+ if (f.isDirectory()) {
+ for (File child : f.listFiles()) {
+ fs.push(child);
+ }
+ } else {
+ System.out.println("Processed " + f);
+ if (f.getName().startsWith("enrichment") || f.getName().endsWith(".json")) {
+ List<byte[]> data = TestUtils.readSampleData(f.getPath());
+ Iterables.addAll(ret, Iterables.transform(data, new Function<byte[], Map<String, Object>>() {
+ @Nullable
+ @Override
+ public Map<String, Object> apply(@Nullable byte[] bytes) {
+ String s = new String(bytes);
+ try {
+ return JSONUtils.INSTANCE.load(s, new TypeReference<Map<String, Object>>() {
+ });
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }));
+ }
+ }
+ }
+ }
+ return ret;
+ }
+
+ public static void cleanHdfsDir(String hdfsDirStr) {
+ File hdfsDir = new File(hdfsDirStr);
+ Stack<File> fs = new Stack<>();
+ if (hdfsDir.exists()) {
+ fs.push(hdfsDir);
+ while (!fs.empty()) {
+ File f = fs.pop();
+ if (f.isDirectory()) {
+ for (File child : f.listFiles()) {
+ fs.push(child);
+ }
+ } else {
+ if (f.getName().startsWith("enrichment") || f.getName().endsWith(".json")) {
+ f.delete();
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ public void test() throws Exception {
+ cleanHdfsDir(hdfsDir);
+ final String dateFormat = "yyyy.MM.dd.HH";
+ final List<byte[]> inputMessages = TestUtils.readSampleData(sampleParsedPath);
+ final Properties topologyProperties = new Properties() {{
+ setProperty("kafka.start", SpoutConfig.Offset.BEGINNING.name());
+ setProperty("indexing.workers", "1");
+ setProperty("indexing.executors", "0");
+ setProperty("index.input.topic", Constants.INDEXING_TOPIC);
+ setProperty("index.error.topic", Constants.INDEXING_ERROR_TOPIC);
+ setProperty("index.date.format", dateFormat);
+ //HDFS settings
+
+ setProperty("bolt.hdfs.rotation.policy", TimedRotationPolicy.class.getCanonicalName());
+ setProperty("bolt.hdfs.rotation.policy.count", "1");
+ setProperty("bolt.hdfs.rotation.policy.units", "DAYS");
+ setProperty("index.hdfs.output", hdfsDir);
+ }};
+ setAdditionalProperties(topologyProperties);
+ final KafkaWithZKComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaWithZKComponent.Topic>() {{
+ add(new KafkaWithZKComponent.Topic(Constants.INDEXING_TOPIC, 1));
+ add(new KafkaWithZKComponent.Topic(Constants.INDEXING_ERROR_TOPIC, 1));
+ }});
+ List<Map<String, Object>> inputDocs = new ArrayList<>();
+ for(byte[] b : inputMessages) {
+ Map<String, Object> m = JSONUtils.INSTANCE.load(new String(b), new TypeReference<Map<String, Object>>() {});
+ inputDocs.add(m);
+
+ }
+ ConfigUploadComponent configUploadComponent = new ConfigUploadComponent()
+ .withTopologyProperties(topologyProperties)
+ .withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH)
+ .withEnrichmentConfigsPath(TestConstants.SAMPLE_CONFIG_PATH);
+ FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder()
+ .withTopologyLocation(new File(fluxPath))
+ .withTopologyName("test")
+ .withTopologyProperties(topologyProperties)
+ .build();
+
+
+ ComponentRunner runner = new ComponentRunner.Builder()
+ .withComponent("kafka", kafkaComponent)
+ .withComponent("config", configUploadComponent)
+ .withComponent("storm", fluxComponent)
+ .withComponent("search", getSearchComponent(topologyProperties))
+ .withMillisecondsBetweenAttempts(15000)
+ .withNumRetries(10)
+ .build();
+ runner.start();
+
+ try {
+ fluxComponent.submitTopology();
+
+ kafkaComponent.writeMessages(Constants.INDEXING_TOPIC, inputMessages);
+ List<Map<String, Object>> docs = cleanDocs(runner.process(getProcessor(inputMessages)));
+ Assert.assertEquals(docs.size(), inputMessages.size());
+ //assert that our input docs are equivalent to the output docs, converting the input docs keys based
+ // on the field name converter
+ assertInputDocsMatchOutputs(inputDocs, docs, getFieldNameConverter());
+ assertInputDocsMatchOutputs(inputDocs, readDocsFromDisk(hdfsDir), x -> x);
+ }
+ finally {
+ if(runner != null) {
+ runner.stop();
+ }
+ }
+ }
+
+ public List<Map<String, Object>> cleanDocs(List<Map<String, Object>> docs) {
+ List<Map<String, Object>> ret = new ArrayList<>();
+ for (Map<String, Object> doc : docs) {
+ Map<String, Object> msg = new HashMap<>();
+ for (Map.Entry<String, Object> kv : doc.entrySet()) {
+ //for writers like solr who modify the keys, we want to undo that if we can
+ msg.put(cleanField(kv.getKey()), kv.getValue());
+ }
+ ret.add(msg);
+ }
+ return ret;
+ }
+
+ public void assertInputDocsMatchOutputs( List<Map<String, Object>> inputDocs
+ , List<Map<String, Object>> indexDocs
+ , FieldNameConverter converter
+ )
+ {
+ for(Map<String, Object> indexDoc : indexDocs) {
+ boolean foundMatch = false;
+ for(Map<String, Object> doc : inputDocs) {
+ if(docMatches(indexDoc, doc, converter)) {
+ foundMatch = true;
+ break;
+ }
+ }
+ if(!foundMatch) {
+ System.err.println("Unable to find: ");
+ printMessage(indexDoc);
+ dumpMessages("INPUT DOCS:", inputDocs);
+ }
+ Assert.assertTrue(foundMatch);
+ }
+ }
+
+ private void printMessage(Map<String, Object> doc) {
+ TreeMap<String, Object> d = new TreeMap<>(doc);
+ for(Map.Entry<String, Object> kv : d.entrySet()) {
+ System.err.println(" " + kv.getKey() + " -> " + kv.getValue());
+ }
+ }
+
+ private void dumpMessages(String title, List<Map<String, Object>> docs) {
+ System.err.println(title);
+ int cnt = 0;
+ for(Map<String, Object> doc : docs) {
+ System.err.println("MESSAGE " + cnt++);
+ printMessage(doc);
+ }
+ }
+
+ boolean docMatches(Map<String, Object> indexedDoc, Map<String, Object> inputDoc, FieldNameConverter converter) {
+ String key = "original_string";
+ String indexKey = converter.convert(key);
+ String originalString = inputDoc.get(key).toString();
+ return originalString.equals(indexedDoc.get(indexKey).toString());
+ }
+ public abstract Processor<List<Map<String, Object>>> getProcessor (List <byte[]>inputMessages);
+ public abstract FieldNameConverter getFieldNameConverter();
+ public abstract InMemoryComponent getSearchComponent(final Properties topologyProperties) throws Exception;
+ public abstract void setAdditionalProperties(Properties topologyProperties);
+ public abstract String cleanField(String field);
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/pom.xml b/metron-platform/metron-integration-test/pom.xml
index 130de5e..4aa10cf 100644
--- a/metron-platform/metron-integration-test/pom.xml
+++ b/metron-platform/metron-integration-test/pom.xml
@@ -143,11 +143,6 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
<artifactId>metron-test-utilities</artifactId>
<version>${project.parent.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java
index 1a3247c..5fdae49 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/BaseIntegrationTest.java
@@ -20,7 +20,6 @@ package org.apache.metron.integration;
import com.google.common.base.Function;
import org.apache.metron.TestConstants;
import org.apache.metron.integration.components.KafkaWithZKComponent;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
import javax.annotation.Nullable;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
deleted file mode 100644
index 483f2b5..0000000
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
+++ /dev/null
@@ -1,489 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.google.common.base.*;
-
-import com.google.common.collect.Iterables;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.metron.common.Constants;
-import org.apache.metron.TestConstants;
-import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.common.interfaces.FieldNameConverter;
-import org.apache.metron.common.configuration.EnrichmentConfigurations;
-import org.apache.metron.hbase.TableProvider;
-import org.apache.metron.enrichment.converter.EnrichmentKey;
-import org.apache.metron.enrichment.converter.EnrichmentValue;
-import org.apache.metron.enrichment.converter.EnrichmentHelper;
-import org.apache.metron.integration.components.ConfigUploadComponent;
-import org.apache.metron.integration.utils.TestUtils;
-import org.apache.metron.test.utils.UnitTestHelper;
-import org.apache.metron.integration.components.FluxTopologyComponent;
-import org.apache.metron.integration.components.KafkaWithZKComponent;
-import org.apache.metron.integration.mock.MockGeoAdapter;
-import org.apache.metron.test.mock.MockHTable;
-import org.apache.metron.enrichment.lookup.LookupKV;
-
-import org.apache.metron.integration.utils.SampleUtil;
-import org.apache.metron.common.utils.JSONUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-import javax.annotation.Nullable;
-import java.io.File;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.Stack;
-
-public abstract class EnrichmentIntegrationTest extends BaseIntegrationTest {
-
- private static final String SRC_IP = "ip_src_addr";
- private static final String DST_IP = "ip_dst_addr";
- private static final String MALICIOUS_IP_TYPE = "malicious_ip";
- private static final String PLAYFUL_CLASSIFICATION_TYPE = "playful_classification";
- private static final Map<String, Object> PLAYFUL_ENRICHMENT = new HashMap<String, Object>() {{
- put("orientation", "north");
- }};
- protected String testSensorType = "test";
- protected String hdfsDir = "target/enrichmentIntegrationTest/hdfs";
- protected String fluxPath = "../metron-enrichment/src/main/flux/enrichment/test.yaml";
- protected String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
- private String sampleIndexedPath = TestConstants.SAMPLE_DATA_INDEXED_PATH + "TestIndexed";
-
- public static class Provider implements TableProvider, Serializable {
- MockHTable.Provider provider = new MockHTable.Provider();
- @Override
- public HTableInterface getTable(Configuration config, String tableName) throws IOException {
- return provider.getTable(config, tableName);
- }
- }
-
- public static void cleanHdfsDir(String hdfsDirStr) {
- File hdfsDir = new File(hdfsDirStr);
- Stack<File> fs = new Stack<>();
- if(hdfsDir.exists()) {
- fs.push(hdfsDir);
- while(!fs.empty()) {
- File f = fs.pop();
- if (f.isDirectory()) {
- for(File child : f.listFiles()) {
- fs.push(child);
- }
- }
- else {
- if (f.getName().startsWith("enrichment") || f.getName().endsWith(".json")) {
- f.delete();
- }
- }
- }
- }
- }
-
- public static List<Map<String, Object> > readDocsFromDisk(String hdfsDirStr) throws IOException {
- List<Map<String, Object>> ret = new ArrayList<>();
- File hdfsDir = new File(hdfsDirStr);
- Stack<File> fs = new Stack<>();
- if(hdfsDir.exists()) {
- fs.push(hdfsDir);
- while(!fs.empty()) {
- File f = fs.pop();
- if(f.isDirectory()) {
- for (File child : f.listFiles()) {
- fs.push(child);
- }
- }
- else {
- System.out.println("Processed " + f);
- if (f.getName().startsWith("enrichment") || f.getName().endsWith(".json")) {
- List<byte[]> data = TestUtils.readSampleData(f.getPath());
- Iterables.addAll(ret, Iterables.transform(data, new Function<byte[], Map<String, Object>>() {
- @Nullable
- @Override
- public Map<String, Object> apply(@Nullable byte[] bytes) {
- String s = new String(bytes);
- try {
- return JSONUtils.INSTANCE.load(s, new TypeReference<Map<String, Object>>() {
- });
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }));
- }
- }
- }
- }
- return ret;
- }
-
-
- @Test
- public void test() throws Exception {
- cleanHdfsDir(hdfsDir);
- final EnrichmentConfigurations configurations = SampleUtil.getSampleEnrichmentConfigs();
- final String dateFormat = "yyyy.MM.dd.HH";
- final List<byte[]> inputMessages = TestUtils.readSampleData(sampleParsedPath);
- final String cf = "cf";
- final String trackerHBaseTableName = "tracker";
- final String threatIntelTableName = "threat_intel";
- final String enrichmentsTableName = "enrichments";
- final Properties topologyProperties = new Properties() {{
- setProperty("org.apache.metron.enrichment.host.known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"},\n" +
- "{\"ip\":\"10.1.128.237\", \"local\":\"UNKNOWN\", \"type\":\"unknown\", \"asset_value\" : \"important\"},\n" +
- "{\"ip\":\"10.60.10.254\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"},\n" +
- "{\"ip\":\"10.0.2.15\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}]");
- setProperty("hbase.provider.impl","" + Provider.class.getName());
- setProperty("threat.intel.tracker.table", trackerHBaseTableName);
- setProperty("threat.intel.tracker.cf", cf);
- setProperty("threat.intel.simple.hbase.table", threatIntelTableName);
- setProperty("threat.intel.simple.hbase.cf", cf);
- setProperty("enrichment.simple.hbase.table", enrichmentsTableName);
- setProperty("enrichment.simple.hbase.cf", cf);
- setProperty("es.clustername", "metron");
- setProperty("es.port", "9300");
- setProperty("es.ip", "localhost");
- setProperty("index.date.format", dateFormat);
- setProperty("index.hdfs.output", hdfsDir);
- }};
- setAdditionalProperties(topologyProperties);
- final KafkaWithZKComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaWithZKComponent.Topic>() {{
- add(new KafkaWithZKComponent.Topic(Constants.ENRICHMENT_TOPIC, 1));
- }});
-
- ConfigUploadComponent configUploadComponent = new ConfigUploadComponent()
- .withTopologyProperties(topologyProperties)
- .withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH)
- .withEnrichmentConfigsPath(TestConstants.SAMPLE_CONFIG_PATH);
-
- //create MockHBaseTables
- final MockHTable trackerTable = (MockHTable)MockHTable.Provider.addToCache(trackerHBaseTableName, cf);
- final MockHTable threatIntelTable = (MockHTable)MockHTable.Provider.addToCache(threatIntelTableName, cf);
- EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>(){{
- add(new LookupKV<>(new EnrichmentKey(MALICIOUS_IP_TYPE, "10.0.2.3"), new EnrichmentValue(new HashMap<>())));
- }});
- final MockHTable enrichmentTable = (MockHTable)MockHTable.Provider.addToCache(enrichmentsTableName, cf);
- EnrichmentHelper.INSTANCE.load(enrichmentTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>(){{
- add(new LookupKV<>(new EnrichmentKey(PLAYFUL_CLASSIFICATION_TYPE, "10.0.2.3")
- , new EnrichmentValue(PLAYFUL_ENRICHMENT )
- )
- );
- }});
- FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder()
- .withTopologyLocation(new File(fluxPath))
- .withTopologyName("test")
- .withTopologyProperties(topologyProperties)
- .build();
-
- InMemoryComponent searchComponent = getSearchComponent(topologyProperties);
-
- UnitTestHelper.verboseLogging();
- ComponentRunner runner = new ComponentRunner.Builder()
- .withComponent("kafka", kafkaComponent)
- .withComponent("config", configUploadComponent)
- .withComponent("search", searchComponent)
- .withComponent("storm", fluxComponent)
- .withMillisecondsBetweenAttempts(15000)
- .withNumRetries(10)
- .build();
- runner.start();
-
- try {
- fluxComponent.submitTopology();
-
- kafkaComponent.writeMessages(Constants.ENRICHMENT_TOPIC, inputMessages);
- List<Map<String, Object>> docs = runner.process(getProcessor(inputMessages));
- Assert.assertEquals(inputMessages.size(), docs.size());
- List<Map<String, Object>> cleanedDocs = cleanDocs(docs);
- validateAll(cleanedDocs, getFieldNameConverter());
-
-
- List<Map<String, Object>> docsFromDisk = readDocsFromDisk(hdfsDir);
- Assert.assertEquals(docsFromDisk.size(), docs.size()) ;
- Assert.assertEquals(new File(hdfsDir).list().length, 1);
- Assert.assertEquals(new File(hdfsDir).list()[0], testSensorType);
- //we want the identity transformation when dealing with docs on disk.
- validateAll(docsFromDisk, fieldName -> fieldName);
- }
- finally {
- cleanHdfsDir(hdfsDir);
- runner.stop();
- }
- }
-
- public List<Map<String, Object>> cleanDocs(List<Map<String, Object>> docs) {
- List<Map<String, Object>> cleanedDocs = new ArrayList<>();
- for(Map<String, Object> doc: docs) {
- Map<String, Object> cleanedFields = new HashMap<>();
- for(String field: doc.keySet()) {
- cleanedFields.put(cleanField(field), doc.get(field));
- }
- cleanedDocs.add(cleanedFields);
- }
- return cleanedDocs;
- }
-
- public static void validateAll(List<Map<String, Object>> docs, FieldNameConverter fnc) {
-
- for (Map<String, Object> doc : docs) {
- baseValidation(doc, fnc);
- hostEnrichmentValidation(doc, fnc);
- geoEnrichmentValidation(doc, fnc);
- threatIntelValidation(doc, fnc);
- simpleEnrichmentValidation(doc, fnc);
- }
- }
-
- public static void baseValidation(Map<String, Object> jsonDoc, FieldNameConverter fnc) {
- assertEnrichmentsExists("threatintels.", setOf("hbaseThreatIntel"), jsonDoc.keySet());
- assertEnrichmentsExists("enrichments.", setOf("geo", "host", "hbaseEnrichment" ), jsonDoc.keySet());
- for(Map.Entry<String, Object> kv : jsonDoc.entrySet()) {
- //ensure no values are empty.
- Assert.assertTrue(kv.getValue().toString().length() > 0);
- }
- //ensure we always have a source ip and destination ip
- Assert.assertNotNull(jsonDoc.get(SRC_IP));
- Assert.assertNotNull(jsonDoc.get(DST_IP));
- }
-
- private static class EvaluationPayload {
- Map<String, Object> indexedDoc;
- String key;
- FieldNameConverter fnc;
- public EvaluationPayload(Map<String, Object> indexedDoc, String key, FieldNameConverter fnc) {
- this.indexedDoc = indexedDoc;
- this.key = key;
- this.fnc = fnc;
- }
- }
-
- private static enum HostEnrichments implements Predicate<EvaluationPayload>{
-
- LOCAL_LOCATION(new Predicate<EvaluationPayload>() {
-
- @Override
- public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
-
- return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.local"),"").equals("YES");
-
- }
- })
-
- ,UNKNOWN_LOCATION(new Predicate<EvaluationPayload>() {
-
- @Override
- public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
- return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.local"),"").equals("UNKNOWN");
- }
- })
- ,IMPORTANT(new Predicate<EvaluationPayload>() {
- @Override
- public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
- return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.asset_value"),"").equals("important");
- }
- })
- ,PRINTER_TYPE(new Predicate<EvaluationPayload>() {
- @Override
- public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
- return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.type"),"").equals("printer");
- }
- })
- ,WEBSERVER_TYPE(new Predicate<EvaluationPayload>() {
- @Override
- public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
- return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.type"),"").equals("webserver");
- }
- })
- ,UNKNOWN_TYPE(new Predicate<EvaluationPayload>() {
- @Override
- public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
- return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.type"),"").equals("unknown");
- }
- })
- ;
-
- Predicate<EvaluationPayload> _predicate;
- HostEnrichments(Predicate<EvaluationPayload> predicate) {
- this._predicate = predicate;
- }
-
- public boolean apply(EvaluationPayload payload) {
- return _predicate.apply(payload);
- }
-
- }
-
- private static void assertEnrichmentsExists(String topLevel, Set<String> expectedEnrichments, Set<String> keys) {
- for(String key : keys) {
- if(key.startsWith(topLevel)) {
- String secondLevel = Iterables.get(Splitter.on(".").split(key), 1);
- String message = "Found an enrichment/threat intel (" + secondLevel + ") that I didn't expect (expected enrichments :"
- + Joiner.on(",").join(expectedEnrichments) + "), but it was not there. If you've created a new"
- + " enrichment, then please add a validation method to this unit test. Otherwise, it's a solid error"
- + " and should be investigated.";
- Assert.assertTrue( message, expectedEnrichments.contains(secondLevel));
- }
- }
- }
- private static void simpleEnrichmentValidation(Map<String, Object> indexedDoc, FieldNameConverter fnc) {
- if(indexedDoc.getOrDefault(fnc.convert(SRC_IP),"").equals("10.0.2.3")
- || indexedDoc.getOrDefault(fnc.convert(DST_IP),"").equals("10.0.2.3")
- ) {
- Assert.assertTrue(keyPatternExists(fnc.convert("enrichments.hbaseEnrichment"), indexedDoc));
- if(indexedDoc.getOrDefault(fnc.convert(SRC_IP),"").equals("10.0.2.3")) {
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.hbaseEnrichment." + SRC_IP + "." + PLAYFUL_CLASSIFICATION_TYPE+ ".orientation"))
- , PLAYFUL_ENRICHMENT.get("orientation")
- );
- }
- else if(indexedDoc.getOrDefault(fnc.convert(DST_IP),"").equals("10.0.2.3")) {
- Assert.assertEquals( indexedDoc.get(fnc.convert("enrichments.hbaseEnrichment." + DST_IP + "." + PLAYFUL_CLASSIFICATION_TYPE + ".orientation"))
- , PLAYFUL_ENRICHMENT.get("orientation")
- );
- }
- }
-
- }
- private static void threatIntelValidation(Map<String, Object> indexedDoc, FieldNameConverter fnc) {
- if(indexedDoc.getOrDefault(fnc.convert(SRC_IP),"").equals("10.0.2.3")
- || indexedDoc.getOrDefault(fnc.convert(DST_IP),"").equals("10.0.2.3")
- ) {
- //if we have any threat intel messages, we want to tag is_alert to true
- Assert.assertTrue(keyPatternExists(fnc.convert("threatintels."), indexedDoc));
- Assert.assertTrue(indexedDoc.containsKey(fnc.convert("threat.triage.level")));
- Assert.assertEquals(indexedDoc.getOrDefault(fnc.convert("is_alert"),""), "true");
- Assert.assertEquals((double)indexedDoc.get(fnc.convert("threat.triage.level")), 10d, 1e-7);
- }
- else {
- //For YAF this is the case, but if we do snort later on, this will be invalid.
- Assert.assertNull(indexedDoc.get(fnc.convert("is_alert")));
- Assert.assertFalse(keyPatternExists(fnc.convert("threatintels."), indexedDoc));
- }
- //ip threat intels
- if(keyPatternExists(fnc.convert("threatintels.hbaseThreatIntel."), indexedDoc)) {
- if(indexedDoc.getOrDefault(fnc.convert(SRC_IP),"").equals("10.0.2.3")) {
- Assert.assertEquals(indexedDoc.get(fnc.convert("threatintels.hbaseThreatIntel." + SRC_IP + "." + MALICIOUS_IP_TYPE)), "alert");
- }
- else if(indexedDoc.getOrDefault(fnc.convert(DST_IP),"").equals("10.0.2.3")) {
- Assert.assertEquals(indexedDoc.get(fnc.convert("threatintels.hbaseThreatIntel." + DST_IP + "." + MALICIOUS_IP_TYPE)), "alert");
- }
- else {
- Assert.fail("There was a threat intels that I did not expect: " + indexedDoc);
- }
- }
-
- }
-
- private static void geoEnrichmentValidation(Map<String, Object> indexedDoc, FieldNameConverter fnc) {
- //should have geo enrichment on every message due to mock geo adapter
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".location_point")), MockGeoAdapter.DEFAULT_LOCATION_POINT);
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP +".location_point")), MockGeoAdapter.DEFAULT_LOCATION_POINT);
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".longitude")), MockGeoAdapter.DEFAULT_LONGITUDE);
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".longitude")), MockGeoAdapter.DEFAULT_LONGITUDE);
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".city")), MockGeoAdapter.DEFAULT_CITY);
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".city")), MockGeoAdapter.DEFAULT_CITY);
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".latitude")), MockGeoAdapter.DEFAULT_LATITUDE);
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".latitude")), MockGeoAdapter.DEFAULT_LATITUDE);
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".country")), MockGeoAdapter.DEFAULT_COUNTRY);
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".country")), MockGeoAdapter.DEFAULT_COUNTRY);
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".dmaCode")), MockGeoAdapter.DEFAULT_DMACODE);
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".dmaCode")), MockGeoAdapter.DEFAULT_DMACODE);
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".postalCode")), MockGeoAdapter.DEFAULT_POSTAL_CODE);
- Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".postalCode")), MockGeoAdapter.DEFAULT_POSTAL_CODE);
- }
-
- private static void hostEnrichmentValidation(Map<String, Object> indexedDoc, FieldNameConverter fnc) {
- boolean enriched = false;
- //important local printers
- {
- Set<String> ips = setOf("10.0.2.15", "10.60.10.254");
- if (ips.contains(indexedDoc.get(fnc.convert(SRC_IP)))) {
- //this is a local, important, printer
- Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
- ,HostEnrichments.IMPORTANT
- ,HostEnrichments.PRINTER_TYPE
- ).apply(new EvaluationPayload(indexedDoc, SRC_IP, fnc))
- );
- enriched = true;
- }
- if (ips.contains(indexedDoc.get(fnc.convert(DST_IP)))) {
- Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
- ,HostEnrichments.IMPORTANT
- ,HostEnrichments.PRINTER_TYPE
- ).apply(new EvaluationPayload(indexedDoc, DST_IP, fnc))
- );
- enriched = true;
- }
- }
- //important local webservers
- {
- Set<String> ips = setOf("10.1.128.236");
- if (ips.contains(indexedDoc.get(fnc.convert(SRC_IP)))) {
- //this is a local, important, printer
- Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
- ,HostEnrichments.IMPORTANT
- ,HostEnrichments.WEBSERVER_TYPE
- ).apply(new EvaluationPayload(indexedDoc, SRC_IP, fnc))
- );
- enriched = true;
- }
- if (ips.contains(indexedDoc.get(fnc.convert(DST_IP)))) {
- Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
- ,HostEnrichments.IMPORTANT
- ,HostEnrichments.WEBSERVER_TYPE
- ).apply(new EvaluationPayload(indexedDoc, DST_IP, fnc))
- );
- enriched = true;
- }
- }
- if(!enriched) {
- Assert.assertFalse(keyPatternExists("enrichments.host", indexedDoc));
- }
- }
-
-
- private static boolean keyPatternExists(String pattern, Map<String, Object> indexedObj) {
- for(String k : indexedObj.keySet()) {
- if(k.startsWith(pattern)) {
- return true;
- }
- }
- return false;
- }
- private static Set<String> setOf(String... items) {
- Set<String> ret = new HashSet<>();
- for(String item : items) {
- ret.add(item);
- }
- return ret;
- }
-
- abstract public FieldNameConverter getFieldNameConverter();
- abstract public InMemoryComponent getSearchComponent(Properties topologyProperties) throws Exception;
- abstract public Processor<List<Map<String, Object>>> getProcessor(List<byte[]> inputMessages);
- abstract public void setAdditionalProperties(Properties topologyProperties);
- abstract public String cleanField(String field);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
deleted file mode 100644
index 7e0f325..0000000
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/components/ConfigUploadComponent.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.components;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
-import org.apache.metron.common.configuration.SensorParserConfig;
-import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.integration.InMemoryComponent;
-import org.apache.metron.integration.UnableToStartException;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Properties;
-
-public class ConfigUploadComponent implements InMemoryComponent {
-
- private Properties topologyProperties;
- private String globalConfigPath;
- private String parserConfigsPath;
- private String enrichmentConfigsPath;
- private Optional<String> globalConfig = Optional.empty();
- private Map<String, SensorParserConfig> parserSensorConfigs = new HashMap<>();
- public ConfigUploadComponent withTopologyProperties(Properties topologyProperties) {
- this.topologyProperties = topologyProperties;
- return this;
- }
-
- public ConfigUploadComponent withGlobalConfigsPath(String globalConfigPath) {
- this.globalConfigPath = globalConfigPath;
- return this;
- }
-
- public ConfigUploadComponent withParserConfigsPath(String parserConfigsPath) {
- this.parserConfigsPath = parserConfigsPath;
- return this;
- }
- public ConfigUploadComponent withEnrichmentConfigsPath(String enrichmentConfigsPath) {
- this.enrichmentConfigsPath = enrichmentConfigsPath;
- return this;
- }
-
- public ConfigUploadComponent withParserSensorConfig(String name, SensorParserConfig config) {
- parserSensorConfigs.put(name, config);
- return this;
- }
-
- public ConfigUploadComponent withGlobalConfig(String globalConfig) {
- this.globalConfig = Optional.ofNullable(globalConfig);
- return this;
- }
-
- @Override
- public void start() throws UnableToStartException {
- try {
- if(globalConfigPath != null) {
- ConfigurationsUtils.uploadConfigsToZookeeper(globalConfigPath
- , parserConfigsPath
- , enrichmentConfigsPath
- , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
- );
- }
- for(Map.Entry<String, SensorParserConfig> kv : parserSensorConfigs.entrySet()) {
- ConfigurationsUtils.writeSensorParserConfigToZookeeper( kv.getKey()
- , kv.getValue()
- , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
- );
- }
- if(globalConfig.isPresent()) {
- ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig.get().getBytes()
- , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
- );
- }
-
- } catch (Exception e) {
- throw new UnableToStartException(e.getMessage(), e);
- }
- }
-
- public SensorParserConfig getSensorParserConfig(String sensorType) {
- SensorParserConfig sensorParserConfig = new SensorParserConfig();
- CuratorFramework client = ConfigurationsUtils.getClient(topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY));
- client.start();
- try {
- sensorParserConfig = ConfigurationsUtils.readSensorParserConfigFromZookeeper(sensorType, client);
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- client.close();
- }
- return sensorParserConfig;
- }
-
- @Override
- public void stop() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockGeoAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockGeoAdapter.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockGeoAdapter.java
deleted file mode 100644
index 7316112..0000000
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockGeoAdapter.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.mock;
-
-import com.google.common.base.Joiner;
-import org.apache.metron.enrichment.bolt.CacheKey;
-import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
-import org.json.simple.JSONObject;
-
-import java.io.Serializable;
-
-public class MockGeoAdapter implements EnrichmentAdapter<CacheKey>,
- Serializable {
-
- public static final String DEFAULT_LOC_ID = "1";
- public static final String DEFAULT_COUNTRY = "test country";
- public static final String DEFAULT_CITY = "test city";
- public static final String DEFAULT_POSTAL_CODE = "test postalCode";
- public static final String DEFAULT_LATITUDE = "test latitude";
- public static final String DEFAULT_LONGITUDE = "test longitude";
- public static final String DEFAULT_DMACODE= "test dmaCode";
- public static final String DEFAULT_LOCATION_POINT= Joiner.on(',').join(DEFAULT_LATITUDE,DEFAULT_LONGITUDE);
-
- @Override
- public void logAccess(CacheKey value) {
-
- }
-
- public JSONObject enrich(CacheKey cache ) {
- JSONObject enriched = new JSONObject();
- enriched.put("locID", DEFAULT_LOC_ID);
- enriched.put("country", DEFAULT_COUNTRY);
- enriched.put("city", DEFAULT_CITY);
- enriched.put("postalCode", DEFAULT_POSTAL_CODE);
- enriched.put("latitude", DEFAULT_LATITUDE);
- enriched.put("longitude", DEFAULT_LONGITUDE);
- enriched.put("dmaCode", DEFAULT_DMACODE);
- enriched.put("location_point", DEFAULT_LOCATION_POINT);
- return enriched;
- }
-
- public boolean initializeAdapter() {
- return true;
- }
-
- public void cleanup() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockHBaseConnector.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockHBaseConnector.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockHBaseConnector.java
deleted file mode 100644
index f40b366..0000000
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockHBaseConnector.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.mock;
-
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
-import org.apache.metron.hbase.Connector;
-import org.apache.metron.hbase.TupleTableConfig;
-
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-public class MockHBaseConnector extends Connector {
- static List<Put> puts = Collections.synchronizedList(new ArrayList<Put>());
- public MockHBaseConnector(TupleTableConfig conf, String _quorum, String _port) throws IOException {
- super(conf, _quorum, _port);
- }
-
- @Override
- public void put(Put put) throws InterruptedIOException, RetriesExhaustedWithDetailsException {
- puts.add(put);
- }
-
- @Override
- public void close() {
-
- }
- public static void clear() {
- puts.clear();
- }
- public static List<Put> getPuts() {
- return puts;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java
deleted file mode 100644
index becadce..0000000
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockTableProvider.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.integration.mock;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.metron.hbase.TableProvider;
-import org.apache.metron.test.mock.MockHTable;
-
-import java.io.IOException;
-import java.io.Serializable;
-
-public class MockTableProvider implements TableProvider, Serializable {
- static MockHTable.Provider provider = new MockHTable.Provider();
- @Override
- public HTableInterface getTable(Configuration config, String tableName) throws IOException {
- return provider.getTable(config, tableName);
- }
- public static void addTable(String tableName, String... cf) {
- provider.addToCache(tableName, cf);
- }
- public static MockHTable getTable(String tableName) {
- try {
- return (MockHTable) provider.getTable(null, tableName);
- } catch (IOException e) {
- throw new RuntimeException("Unable to get table: " + tableName);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/SampleUtil.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/SampleUtil.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/SampleUtil.java
deleted file mode 100644
index 8e293c0..0000000
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/SampleUtil.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.integration.utils;
-
-import org.apache.metron.TestConstants;
-import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
-import org.apache.metron.common.configuration.EnrichmentConfigurations;
-import org.apache.metron.common.configuration.ParserConfigurations;
-
-import java.io.IOException;
-import java.util.Map;
-
-public class SampleUtil {
-
- public static Configurations getSampleConfigs() throws IOException {
- Configurations configurations = new Configurations();
- configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
- return configurations;
- }
-
- public static ParserConfigurations getSampleParserConfigs() throws IOException {
- ParserConfigurations configurations = new ParserConfigurations();
- configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
- Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH);
- for(String sensorType: sensorParserConfigs.keySet()) {
- configurations.updateSensorParserConfig(sensorType, sensorParserConfigs.get(sensorType));
- }
- return configurations;
- }
-
- public static EnrichmentConfigurations getSampleEnrichmentConfigs() throws IOException {
- EnrichmentConfigurations configurations = new EnrichmentConfigurations();
- configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
- Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.SAMPLE_CONFIG_PATH);
- for(String sensorType: sensorEnrichmentConfigs.keySet()) {
- configurations.updateSensorEnrichmentConfig(sensorType, sensorEnrichmentConfigs.get(sensorType));
- }
- return configurations;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index c801a7f..47f7bdd 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -33,8 +33,20 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
+ <artifactId>metron-writer</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-enrichment</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
<artifactId>metron-enrichment</artifactId>
<version>${project.parent.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
@@ -108,12 +120,12 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>2.2.3</version>
+ <version>${global_jackson_version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
- <version>2.2.3</version>
+ <version>${global_jackson_version}</version>
</dependency>
<dependency>
<groupId>io.thekraken</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/asa/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/asa/remote.yaml b/metron-platform/metron-parsers/src/main/flux/asa/remote.yaml
index 052728e..828e1a5 100644
--- a/metron-platform/metron-parsers/src/main/flux/asa/remote.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/asa/remote.yaml
@@ -23,7 +23,7 @@ components:
- id: "parser"
className: "org.apache.metron.parsers.asa.GrokAsaParser"
- id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
constructorArgs:
- "${kafka.broker}"
- id: "zkHosts"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/asa/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/asa/test.yaml b/metron-platform/metron-parsers/src/main/flux/asa/test.yaml
index c816b45..4694fff 100644
--- a/metron-platform/metron-parsers/src/main/flux/asa/test.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/asa/test.yaml
@@ -23,7 +23,7 @@ components:
- id: "parser"
className: "org.apache.metron.parsers.asa.GrokAsaParser"
- id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
constructorArgs:
- "${kafka.broker}"
- id: "zkHosts"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/fireeye/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/fireeye/remote.yaml b/metron-platform/metron-parsers/src/main/flux/fireeye/remote.yaml
index a745d38..dc1ff6b 100644
--- a/metron-platform/metron-parsers/src/main/flux/fireeye/remote.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/fireeye/remote.yaml
@@ -22,7 +22,7 @@ components:
- id: "parser"
className: "org.apache.metron.parsers.fireeye.BasicFireEyeParser"
- id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
constructorArgs:
- "${kafka.broker}"
- id: "zkHosts"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/fireeye/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/fireeye/test.yaml b/metron-platform/metron-parsers/src/main/flux/fireeye/test.yaml
index 9f4c06f..f676b2a 100644
--- a/metron-platform/metron-parsers/src/main/flux/fireeye/test.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/fireeye/test.yaml
@@ -22,7 +22,7 @@ components:
- id: "parser"
className: "org.apache.metron.parsers.fireeye.BasicFireEyeParser"
- id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
constructorArgs:
- "${kafka.broker}"
- id: "zkHosts"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/ise/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/ise/remote.yaml b/metron-platform/metron-parsers/src/main/flux/ise/remote.yaml
index 53ed5fc..49c4c19 100644
--- a/metron-platform/metron-parsers/src/main/flux/ise/remote.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/ise/remote.yaml
@@ -22,7 +22,7 @@ components:
- id: "parser"
className: "org.apache.metron.parsers.ise.BasicIseParser"
- id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
constructorArgs:
- "${kafka.broker}"
- id: "zkHosts"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/ise/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/ise/test.yaml b/metron-platform/metron-parsers/src/main/flux/ise/test.yaml
index 2e88594..8b292e0 100644
--- a/metron-platform/metron-parsers/src/main/flux/ise/test.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/ise/test.yaml
@@ -22,7 +22,7 @@ components:
- id: "parser"
className: "org.apache.metron.parsers.ise.BasicIseParser"
- id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
constructorArgs:
- "${kafka.broker}"
- id: "zkHosts"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/lancope/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/lancope/remote.yaml b/metron-platform/metron-parsers/src/main/flux/lancope/remote.yaml
index 4fe7c3b..f60e86a 100644
--- a/metron-platform/metron-parsers/src/main/flux/lancope/remote.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/lancope/remote.yaml
@@ -22,7 +22,7 @@ components:
- id: "parser"
className: "org.apache.metron.parsers.lancope.BasicLancopeParser"
- id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
constructorArgs:
- "${kafka.broker}"
- id: "zkHosts"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/lancope/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/lancope/test.yaml b/metron-platform/metron-parsers/src/main/flux/lancope/test.yaml
index eb8a1ef..f6ade8e 100644
--- a/metron-platform/metron-parsers/src/main/flux/lancope/test.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/lancope/test.yaml
@@ -22,7 +22,7 @@ components:
- id: "parser"
className: "org.apache.metron.parsers.lancope.BasicLancopeParser"
- id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
constructorArgs:
- "${kafka.broker}"
- id: "zkHosts"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/paloalto/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/paloalto/remote.yaml b/metron-platform/metron-parsers/src/main/flux/paloalto/remote.yaml
index 4287fce..74ea3ad 100644
--- a/metron-platform/metron-parsers/src/main/flux/paloalto/remote.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/paloalto/remote.yaml
@@ -22,7 +22,7 @@ components:
- id: "parser"
className: "org.apache.metron.parsers.paloalto.BasicPaloAltoFirewallParser"
- id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
constructorArgs:
- "${kafka.broker}"
- id: "zkHosts"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/paloalto/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/paloalto/test.yaml b/metron-platform/metron-parsers/src/main/flux/paloalto/test.yaml
index cef5dc5..160b0df 100644
--- a/metron-platform/metron-parsers/src/main/flux/paloalto/test.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/paloalto/test.yaml
@@ -22,7 +22,7 @@ components:
- id: "parser"
className: "org.apache.metron.parsers.paloalto.BasicPaloAltoFirewallParser"
- id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
constructorArgs:
- "${kafka.broker}"
- id: "zkHosts"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/sourcefire/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/sourcefire/remote.yaml b/metron-platform/metron-parsers/src/main/flux/sourcefire/remote.yaml
index 312dce0..a79bfde 100644
--- a/metron-platform/metron-parsers/src/main/flux/sourcefire/remote.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/sourcefire/remote.yaml
@@ -22,7 +22,7 @@ components:
- id: "parser"
className: "org.apache.metron.parsers.sourcefire.BasicSourcefireParser"
- id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
constructorArgs:
- "${kafka.broker}"
- id: "zkHosts"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/sourcefire/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/sourcefire/test.yaml b/metron-platform/metron-parsers/src/main/flux/sourcefire/test.yaml
index a0a00d0..5a6516d 100644
--- a/metron-platform/metron-parsers/src/main/flux/sourcefire/test.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/sourcefire/test.yaml
@@ -22,7 +22,7 @@ components:
- id: "parser"
className: "org.apache.metron.parsers.sourcefire.BasicSourcefireParser"
- id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
constructorArgs:
- "${kafka.broker}"
- id: "zkHosts"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/websphere/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/websphere/remote.yaml b/metron-platform/metron-parsers/src/main/flux/websphere/remote.yaml
index 7d1c103..9f925dd 100644
--- a/metron-platform/metron-parsers/src/main/flux/websphere/remote.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/websphere/remote.yaml
@@ -32,7 +32,7 @@ components:
args:
- "yyyy MMM dd HH:mm:ss"
- id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
constructorArgs:
- "${kafka.broker}"
- id: "zkHosts"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/flux/websphere/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/websphere/test.yaml b/metron-platform/metron-parsers/src/main/flux/websphere/test.yaml
index 045fc0e..20ac9ca 100644
--- a/metron-platform/metron-parsers/src/main/flux/websphere/test.yaml
+++ b/metron-platform/metron-parsers/src/main/flux/websphere/test.yaml
@@ -32,7 +32,7 @@ components:
args:
- "yyyy MMM dd HH:mm:ss"
- id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
constructorArgs:
- "${kafka.broker}"
- id: "zkHosts"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
index aebb8d7..6c40768 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
@@ -26,29 +26,19 @@ import backtype.storm.tuple.Values;
import org.apache.metron.common.Constants;
import org.apache.metron.common.bolt.ConfiguredParserBolt;
import org.apache.metron.common.configuration.FieldValidator;
-import org.apache.metron.common.configuration.ParserConfigurations;
-import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
-import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFacade;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.interfaces.BulkMessageWriter;
-import org.apache.metron.common.writer.BulkWriterComponent;
-import org.apache.metron.common.writer.WriterToBulkWriter;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.parsers.filters.Filters;
import org.apache.metron.common.configuration.FieldTransformer;
-import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.parsers.filters.GenericMessageFilter;
import org.apache.metron.common.utils.ErrorUtils;
import org.apache.metron.parsers.interfaces.MessageFilter;
import org.apache.metron.parsers.interfaces.MessageParser;
-import org.apache.metron.common.interfaces.MessageWriter;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.util.*;
-import java.util.function.Function;
public class ParserBolt extends ConfiguredParserBolt implements Serializable {
[2/5] incubator-metron git commit: METRON-154: Decouple enrichment
and indexing closes apache/incubator-metron#192
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java
index ecd1ce8..e7ad755 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/WriterHandler.java
@@ -26,8 +26,8 @@ import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFac
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.interfaces.BulkMessageWriter;
import org.apache.metron.common.interfaces.MessageWriter;
-import org.apache.metron.common.writer.BulkWriterComponent;
-import org.apache.metron.common.writer.WriterToBulkWriter;
+import org.apache.metron.writer.BulkWriterComponent;
+import org.apache.metron.writer.WriterToBulkWriter;
import org.json.simple.JSONObject;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
index d9004d1..000b7ff 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
@@ -29,12 +29,12 @@ import org.apache.metron.common.interfaces.MessageWriter;
import org.apache.metron.common.spout.kafka.SpoutConfig;
import org.apache.metron.common.spout.kafka.SpoutConfigOptions;
import org.apache.metron.common.utils.ReflectionUtils;
-import org.apache.metron.common.writer.AbstractWriter;
import org.apache.metron.parsers.bolt.ParserBolt;
import org.apache.metron.parsers.bolt.WriterBolt;
import org.apache.metron.parsers.bolt.WriterHandler;
import org.apache.metron.parsers.interfaces.MessageParser;
-import org.apache.metron.parsers.writer.KafkaWriter;
+import org.apache.metron.writer.AbstractWriter;
+import org.apache.metron.writer.kafka.KafkaWriter;
import org.json.simple.JSONObject;
import storm.kafka.KafkaSpout;
import storm.kafka.ZkHosts;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
deleted file mode 100644
index 6090491..0000000
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/writer/KafkaWriter.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers.writer;
-
-import backtype.storm.tuple.Tuple;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.interfaces.MessageWriter;
-import org.apache.metron.common.utils.ConversionUtils;
-import org.apache.metron.common.utils.StringUtils;
-import org.apache.metron.common.writer.AbstractWriter;
-import org.json.simple.JSONObject;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-
-public class KafkaWriter extends AbstractWriter implements MessageWriter<JSONObject>, Serializable {
- public enum Configurations {
- BROKER("kafka.brokerUrl")
- ,KEY_SERIALIZER("kafka.keySerializer")
- ,VALUE_SERIALIZER("kafka.valueSerializer")
- ,REQUIRED_ACKS("kafka.requiredAcks")
- ,TOPIC("kafka.topic")
- ,PRODUCER_CONFIGS("kafka.producerConfigs");
- ;
- String key;
- Configurations(String key) {
- this.key = key;
- }
- public Object get(Optional<String> configPrefix, Map<String, Object> config) {
- return config.get(StringUtils.join(".", configPrefix, Optional.of(key)));
- }
- public <T> T getAndConvert(Optional<String> configPrefix, Map<String, Object> config, Class<T> clazz) {
- Object o = get(configPrefix, config);
- if(o != null) {
- return ConversionUtils.convert(o, clazz);
- }
- return null;
- }
- }
- private String brokerUrl;
- private String keySerializer = "org.apache.kafka.common.serialization.StringSerializer";
- private String valueSerializer = "org.apache.kafka.common.serialization.StringSerializer";
- private int requiredAcks = 1;
- private String kafkaTopic = Constants.ENRICHMENT_TOPIC;
- private KafkaProducer kafkaProducer;
- private String configPrefix = null;
- private Map<String, Object> producerConfigs = new HashMap<>();
-
- public KafkaWriter() {}
-
- public KafkaWriter(String brokerUrl) {
- this.brokerUrl = brokerUrl;
- }
-
- public KafkaWriter withKeySerializer(String keySerializer) {
- this.keySerializer = keySerializer;
- return this;
- }
-
- public KafkaWriter withValueSerializer(String valueSerializer) {
- this.valueSerializer = valueSerializer;
- return this;
- }
-
- public KafkaWriter withRequiredAcks(Integer requiredAcks) {
- this.requiredAcks = requiredAcks;
- return this;
- }
-
- public KafkaWriter withTopic(String topic) {
- this.kafkaTopic= topic;
- return this;
- }
- public KafkaWriter withConfigPrefix(String prefix) {
- this.configPrefix = prefix;
- return this;
- }
-
- public KafkaWriter withProducerConfigs(Map<String, Object> extraConfigs) {
- this.producerConfigs = extraConfigs;
- return this;
- }
-
- public Optional<String> getConfigPrefix() {
- return Optional.ofNullable(configPrefix);
- }
-
- @Override
- public void configure(String sensorName, WriterConfiguration configuration) {
- Map<String, Object> configMap = configuration.getSensorConfig(sensorName);
- String brokerUrl = Configurations.BROKER.getAndConvert(getConfigPrefix(), configMap, String.class);
- if(brokerUrl != null) {
- this.brokerUrl = brokerUrl;
- }
- String keySerializer = Configurations.KEY_SERIALIZER.getAndConvert(getConfigPrefix(), configMap, String.class);
- if(keySerializer != null) {
- withKeySerializer(keySerializer);
- }
- String valueSerializer = Configurations.VALUE_SERIALIZER.getAndConvert(getConfigPrefix(), configMap, String.class);
- if(valueSerializer != null) {
- withValueSerializer(keySerializer);
- }
- Integer requiredAcks = Configurations.REQUIRED_ACKS.getAndConvert(getConfigPrefix(), configMap, Integer.class);
- if(requiredAcks!= null) {
- withRequiredAcks(requiredAcks);
- }
- String topic = Configurations.TOPIC.getAndConvert(getConfigPrefix(), configMap, String.class);
- if(topic != null) {
- withTopic(topic);
- }
- Map<String, Object> producerConfigs = (Map)Configurations.PRODUCER_CONFIGS.get(getConfigPrefix(), configMap);
- if(producerConfigs != null) {
- withProducerConfigs(producerConfigs);
- }
- }
-
- public Map<String, Object> createProducerConfigs() {
- Map<String, Object> producerConfig = new HashMap<>();
- producerConfig.put("bootstrap.servers", brokerUrl);
- producerConfig.put("key.serializer", keySerializer);
- producerConfig.put("value.serializer", valueSerializer);
- producerConfig.put("request.required.acks", requiredAcks);
- producerConfig.putAll(producerConfigs == null?new HashMap<>():producerConfigs);
- return producerConfig;
- }
-
- @Override
- public void init() {
-
- this.kafkaProducer = new KafkaProducer<>(createProducerConfigs());
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void write(String sourceType, WriterConfiguration configurations, Tuple tuple, JSONObject message) throws Exception {
- kafkaProducer.send(new ProducerRecord<String, String>(kafkaTopic, message.toJSONString()));
- }
-
- @Override
- public void close() throws Exception {
- kafkaProducer.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
index 116e262..b3d09d0 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
@@ -18,25 +18,21 @@
package org.apache.metron.parsers.integration;
import junit.framework.Assert;
-import org.apache.commons.io.FilenameUtils;
import org.apache.metron.TestConstants;
import org.apache.metron.common.Constants;
+import org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
import org.apache.metron.integration.BaseIntegrationTest;
import org.apache.metron.integration.ComponentRunner;
import org.apache.metron.integration.Processor;
import org.apache.metron.integration.ReadinessState;
-import org.apache.metron.integration.components.ConfigUploadComponent;
import org.apache.metron.integration.components.KafkaWithZKComponent;
import org.apache.metron.integration.utils.TestUtils;
import org.apache.metron.parsers.integration.components.ParserTopologyComponent;
-import org.apache.metron.parsers.integration.validation.SampleDataValidation;
import org.apache.metron.test.TestDataType;
import org.apache.metron.test.utils.SampleDataUtils;
import org.apache.metron.test.utils.UnitTestHelper;
-import org.junit.Before;
import org.junit.Test;
-import java.io.File;
import java.util.*;
public abstract class ParserIntegrationTest extends BaseIntegrationTest {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/writer/KafkaWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/writer/KafkaWriterTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/writer/KafkaWriterTest.java
deleted file mode 100644
index 57dc3e2..0000000
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/writer/KafkaWriterTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.parsers.writer;
-
-import com.google.common.collect.ImmutableMap;
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.test.TestingServer;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
-import org.apache.metron.common.configuration.ParserConfigurations;
-import org.apache.metron.common.configuration.SensorParserConfig;
-import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-public class KafkaWriterTest {
-
-
- public static final String SENSOR_TYPE = "test";
- public WriterConfiguration createConfiguration(final Map<String, Object> parserConfig) {
- ParserConfigurations configurations = new ParserConfigurations();
- configurations.updateSensorParserConfig( SENSOR_TYPE
- , new SensorParserConfig() {{
- setParserConfig(parserConfig);
- }}
- );
- return new ParserWriterConfiguration(configurations);
- }
-
- @Test
- public void testHappyPathGlobalConfig() throws Exception {
- KafkaWriter writer = new KafkaWriter();
- WriterConfiguration configuration = createConfiguration(
- new HashMap<String, Object>() {{
- put("kafka.brokerUrl" , "localhost:6667");
- put("kafka.topic" , SENSOR_TYPE);
- put("kafka.producerConfigs" , ImmutableMap.of("key1", 1, "key2", "value2"));
- }}
- );
-
- writer.configure(SENSOR_TYPE, configuration);
- Map<String, Object> producerConfigs = writer.createProducerConfigs();
- Assert.assertEquals(producerConfigs.get("bootstrap.servers"), "localhost:6667");
- Assert.assertEquals(producerConfigs.get("key.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
- Assert.assertEquals(producerConfigs.get("value.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
- Assert.assertEquals(producerConfigs.get("request.required.acks"), 1);
- Assert.assertEquals(producerConfigs.get("key1"), 1);
- Assert.assertEquals(producerConfigs.get("key2"), "value2");
- }
-
- @Test
- public void testHappyPathGlobalConfigWithPrefix() throws Exception {
- KafkaWriter writer = new KafkaWriter();
- writer.withConfigPrefix("prefix");
- WriterConfiguration configuration = createConfiguration(
- new HashMap<String, Object>() {{
- put("prefix.kafka.brokerUrl" , "localhost:6667");
- put("prefix.kafka.topic" , SENSOR_TYPE);
- put("prefix.kafka.producerConfigs" , ImmutableMap.of("key1", 1, "key2", "value2"));
- }}
- );
-
- writer.configure(SENSOR_TYPE, configuration);
- Map<String, Object> producerConfigs = writer.createProducerConfigs();
- Assert.assertEquals(producerConfigs.get("bootstrap.servers"), "localhost:6667");
- Assert.assertEquals(producerConfigs.get("key.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
- Assert.assertEquals(producerConfigs.get("value.serializer"), "org.apache.kafka.common.serialization.StringSerializer");
- Assert.assertEquals(producerConfigs.get("request.required.acks"), 1);
- Assert.assertEquals(producerConfigs.get("key1"), 1);
- Assert.assertEquals(producerConfigs.get("key2"), "value2");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
index cea635f..a21d45b 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/SimpleHBaseEnrichmentWriterTest.java
@@ -20,17 +20,16 @@ package org.apache.metron.writers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.enrichment.converter.EnrichmentConverter;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.integration.mock.MockTableProvider;
import org.apache.metron.enrichment.lookup.LookupKV;
-import org.apache.metron.integration.mock.MockTableProvider;
import org.apache.metron.test.mock.MockHTable;
-import org.apache.metron.writer.hbase.SimpleHbaseEnrichmentWriter;
+import org.apache.metron.enrichment.writer.SimpleHbaseEnrichmentWriter;
import org.json.simple.JSONObject;
import org.junit.Assert;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
index f6436b9..2f359ef 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/SimpleHbaseEnrichmentWriterIntegrationTest.java
@@ -20,31 +20,25 @@ package org.apache.metron.writers.integration;
import com.google.common.collect.ImmutableList;
import org.adrianwalker.multilinestring.Multiline;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.metron.TestConstants;
-import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.enrichment.converter.EnrichmentConverter;
import org.apache.metron.enrichment.converter.EnrichmentKey;
import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
+import org.apache.metron.enrichment.integration.mock.MockTableProvider;
import org.apache.metron.enrichment.lookup.LookupKV;
import org.apache.metron.integration.*;
-import org.apache.metron.integration.components.ConfigUploadComponent;
import org.apache.metron.integration.components.KafkaWithZKComponent;
-import org.apache.metron.integration.mock.MockTableProvider;
-import org.apache.metron.integration.utils.TestUtils;
import org.apache.metron.parsers.integration.components.ParserTopologyComponent;
-import org.apache.metron.test.TestDataType;
import org.apache.metron.test.mock.MockHTable;
-import org.apache.metron.test.utils.SampleDataUtils;
import org.apache.metron.test.utils.UnitTestHelper;
import org.junit.Assert;
import org.junit.Test;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.*;
@@ -53,7 +47,7 @@ public class SimpleHbaseEnrichmentWriterIntegrationTest extends BaseIntegrationT
/**
{
"parserClassName" : "org.apache.metron.parsers.csv.CSVParser"
- ,"writerClassName" : "org.apache.metron.writer.hbase.SimpleHbaseEnrichmentWriter"
+ ,"writerClassName" : "org.apache.metron.enrichment.writer.SimpleHbaseEnrichmentWriter"
,"sensorTopic":"dummy"
,"parserConfig":
{
@@ -61,7 +55,7 @@ public class SimpleHbaseEnrichmentWriterIntegrationTest extends BaseIntegrationT
,"shew.cf" : "cf"
,"shew.keyColumns" : "col2"
,"shew.enrichmentType" : "et"
- ,"shew.hbaseProvider" : "org.apache.metron.integration.mock.MockTableProvider"
+ ,"shew.hbaseProvider" : "org.apache.metron.enrichment.integration.mock.MockTableProvider"
,"columns" : {
"col1" : 0
,"col2" : 1
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
index a6e83f1..afdcf04 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
@@ -18,37 +18,22 @@
package org.apache.metron.writers.integration;
import com.fasterxml.jackson.core.type.TypeReference;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import org.adrianwalker.multilinestring.Multiline;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.metron.TestConstants;
import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.FieldValidator;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.field.validation.FieldValidation;
import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.enrichment.converter.EnrichmentConverter;
-import org.apache.metron.enrichment.converter.EnrichmentKey;
-import org.apache.metron.enrichment.converter.EnrichmentValue;
-import org.apache.metron.enrichment.lookup.LookupKV;
+import org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
import org.apache.metron.integration.*;
-import org.apache.metron.integration.components.ConfigUploadComponent;
import org.apache.metron.integration.components.KafkaWithZKComponent;
-import org.apache.metron.integration.mock.MockTableProvider;
-import org.apache.metron.integration.utils.TestUtils;
import org.apache.metron.parsers.integration.components.ParserTopologyComponent;
-import org.apache.metron.test.TestDataType;
-import org.apache.metron.test.mock.MockHTable;
-import org.apache.metron.test.utils.SampleDataUtils;
import org.apache.metron.test.utils.UnitTestHelper;
import org.json.simple.JSONObject;
import org.junit.Assert;
import org.junit.Test;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.*;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-solr/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml
index 54b50f4..3aceda0 100644
--- a/metron-platform/metron-solr/pom.xml
+++ b/metron-platform/metron-solr/pom.xml
@@ -32,16 +32,16 @@
<version>${global_hbase_guava_version}</version>
</dependency>
<dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<version>${global_solr_version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-enrichment</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-test-framework</artifactId>
<version>${global_solr_version}</version>
@@ -98,6 +98,10 @@
<artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>commons-httpclient</artifactId>
+ <groupId>commons-httpclient</groupId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -122,8 +126,8 @@
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -133,6 +137,37 @@
<version>${global_mockito_version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-indexing</artifactId>
+ <version>${project.parent.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-enrichment</artifactId>
+ <version>${project.parent.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-indexing</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
@@ -201,6 +236,7 @@
<goal>shade</goal>
</goals>
<configuration>
+
<artifactSet>
<excludes>
<exclude>storm:storm-core:*</exclude>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-solr/src/main/config/solr.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/solr.properties b/metron-platform/metron-solr/src/main/config/solr.properties
index cdfe25a..8489d8e 100644
--- a/metron-platform/metron-solr/src/main/config/solr.properties
+++ b/metron-platform/metron-solr/src/main/config/solr.properties
@@ -14,37 +14,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+##### Storm #####
+indexing.workers=1
+indexing.executors=0
##### Kafka #####
kafka.zk=node1:2181
kafka.broker=node1:6667
-spout.kafka.topic.asa=asa
-spout.kafka.topic.bro=bro
-spout.kafka.topic.fireeye=fireeye
-spout.kafka.topic.ise=ise
-spout.kafka.topic.lancope=lancope
-spout.kafka.topic.paloalto=paloalto
-spout.kafka.topic.pcap=pcap
-spout.kafka.topic.snort=snort
-spout.kafka.topic.yaf=yaf
+kafka.start=WHERE_I_LEFT_OFF
##### Indexing #####
+index.input.topic=indexing
+index.error.topic=indexing_error
writer.class.name=org.apache.metron.solr.writer.SolrWriter
-##### ElasticSearch #####
-
-es.ip=10.22.0.214
-es.port=9300
-es.clustername=elasticsearch
-
-##### MySQL #####
-
-mysql.ip=10.22.0.214
-mysql.port=3306
-mysql.username=root
-mysql.password=hadoop123
-
##### Metrics #####
#reporters
@@ -63,23 +47,6 @@ org.apache.metron.metrics.TelemetryParserBolt.emits=true
org.apache.metron.metrics.TelemetryParserBolt.fails=true
-#GenericEnrichmentBolt
-org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
-org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
-org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
-
-
-#TelemetryIndexingBolt
-org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
-org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
-org.apache.metron.metrics.TelemetryIndexingBolt.fails=true
-
-##### Host Enrichment #####
-
-org.apache.metron.enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
-{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
-{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
-
##### HDFS #####
bolt.hdfs.batch.size=5000
@@ -91,19 +58,3 @@ bolt.hdfs.finished.file.path=/paloalto/rotated
bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
index.hdfs.output=/tmp/metron/enriched
-##### HBase #####
-bolt.hbase.table.name=pcap
-bolt.hbase.table.fields=t:value
-bolt.hbase.table.key.tuple.field.name=key
-bolt.hbase.table.timestamp.tuple.field.name=timestamp
-bolt.hbase.enable.batching=false
-bolt.hbase.write.buffer.size.in.bytes=2000000
-bolt.hbase.durability=SKIP_WAL
-bolt.hbase.partitioner.region.info.refresh.interval.mins=60
-
-##### Threat Intel #####
-
-threat.intel.tracker.table=
-threat.intel.tracker.cf=
-threat.intel.ip.table=
-threat.intel.ip.cf=
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-solr/src/main/scripts/start_solr_topology.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/scripts/start_solr_topology.sh b/metron-platform/metron-solr/src/main/scripts/start_solr_topology.sh
index 14628d2..7a98fc7 100755
--- a/metron-platform/metron-solr/src/main/scripts/start_solr_topology.sh
+++ b/metron-platform/metron-solr/src/main/scripts/start_solr_topology.sh
@@ -19,4 +19,4 @@
METRON_VERSION=${project.version}
METRON_HOME=/usr/metron/$METRON_VERSION
TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION.jar
-storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/enrichment/remote.yaml --filter $METRON_HOME/config/solr.properties
+storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/indexing/remote.yaml --filter $METRON_HOME/config/solr.properties
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrEnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrEnrichmentIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrEnrichmentIntegrationTest.java
deleted file mode 100644
index 31e8d49..0000000
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrEnrichmentIntegrationTest.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.solr.integration;
-
-import com.google.common.base.Function;
-import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.common.interfaces.FieldNameConverter;
-import org.apache.metron.integration.EnrichmentIntegrationTest;
-import org.apache.metron.integration.ComponentRunner;
-import org.apache.metron.integration.InMemoryComponent;
-import org.apache.metron.integration.Processor;
-import org.apache.metron.integration.ReadinessState;
-import org.apache.metron.integration.components.KafkaWithZKComponent;
-import org.apache.metron.solr.integration.components.SolrComponent;
-import org.apache.metron.integration.utils.SampleUtil;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
-import org.apache.metron.common.utils.JSONUtils;
-
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-public class SolrEnrichmentIntegrationTest extends EnrichmentIntegrationTest {
-
- private String collection = "metron";
- private FieldNameConverter fieldNameConverter = fieldName -> fieldName;
- @Override
- public FieldNameConverter getFieldNameConverter() {
- return fieldNameConverter;
- }
-
- @Override
- public InMemoryComponent getSearchComponent(final Properties topologyProperties) throws Exception {
- SolrComponent solrComponent = new SolrComponent.Builder()
- .addCollection(collection, "../metron-solr/src/test/resources/solr/conf")
- .withPostStartCallback(new Function<SolrComponent, Void>() {
- @Nullable
- @Override
- public Void apply(@Nullable SolrComponent solrComponent) {
- topologyProperties.setProperty("solr.zk", solrComponent.getZookeeperUrl());
- try {
- String testZookeeperUrl = topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY);
- Configurations configurations = SampleUtil.getSampleConfigs();
- Map<String, Object> globalConfig = configurations.getGlobalConfig();
- globalConfig.put("solr.zookeeper", solrComponent.getZookeeperUrl());
- ConfigurationsUtils.writeGlobalConfigToZookeeper(JSONUtils.INSTANCE.toJSON(globalConfig), testZookeeperUrl);
- } catch (Exception e) {
- e.printStackTrace();
- }
- return null;
- }
- })
- .build();
- return solrComponent;
- }
-
- @Override
- public Processor<List<Map<String, Object>>> getProcessor(final List<byte[]> inputMessages) {
- return new Processor<List<Map<String, Object>>>() {
- List<Map<String, Object>> docs = null;
- public ReadinessState process(ComponentRunner runner) {
- SolrComponent solrComponent = runner.getComponent("search", SolrComponent.class);
- if (solrComponent.hasCollection(collection)) {
- List<Map<String, Object>> docsFromDisk;
- try {
- docs = solrComponent.getAllIndexedDocs(collection);
- docsFromDisk = EnrichmentIntegrationTest.readDocsFromDisk(hdfsDir);
- System.out.println(docs.size() + " vs " + inputMessages.size() + " vs " + docsFromDisk.size());
- } catch (IOException e) {
- throw new IllegalStateException("Unable to retrieve indexed documents.", e);
- }
- if (docs.size() < inputMessages.size() || docs.size() != docsFromDisk.size()) {
- return ReadinessState.NOT_READY;
- } else {
- return ReadinessState.READY;
- }
- } else {
- return ReadinessState.NOT_READY;
- }
- }
-
- public List<Map<String, Object>> getResult() {
- return docs;
- }
- };
- }
-
- @Override
- public void setAdditionalProperties(Properties topologyProperties) {
- topologyProperties.setProperty("writer.class.name", "org.apache.metron.solr.writer.SolrWriter");
- }
-
- @Override
- public String cleanField(String field) {
- return field.replaceFirst("_[dfils]$", "");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
new file mode 100644
index 0000000..dd9c559
--- /dev/null
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.solr.integration;
+
+import com.google.common.base.Function;
+import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.interfaces.FieldNameConverter;
+import org.apache.metron.enrichment.integration.utils.SampleUtil;
+import org.apache.metron.indexing.integration.IndexingIntegrationTest;
+import org.apache.metron.integration.ComponentRunner;
+import org.apache.metron.integration.InMemoryComponent;
+import org.apache.metron.integration.Processor;
+import org.apache.metron.integration.ReadinessState;
+import org.apache.metron.integration.components.KafkaWithZKComponent;
+import org.apache.metron.solr.integration.components.SolrComponent;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.utils.JSONUtils;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class SolrIndexingIntegrationTest extends IndexingIntegrationTest {
+
+ private String collection = "metron";
+ private FieldNameConverter fieldNameConverter = fieldName -> fieldName;
+ @Override
+ public FieldNameConverter getFieldNameConverter() {
+ return fieldNameConverter;
+ }
+
+ @Override
+ public InMemoryComponent getSearchComponent(final Properties topologyProperties) throws Exception {
+ SolrComponent solrComponent = new SolrComponent.Builder()
+ .addCollection(collection, "../metron-solr/src/test/resources/solr/conf")
+ .withPostStartCallback(new Function<SolrComponent, Void>() {
+ @Nullable
+ @Override
+ public Void apply(@Nullable SolrComponent solrComponent) {
+ topologyProperties.setProperty("solr.zk", solrComponent.getZookeeperUrl());
+ try {
+ String testZookeeperUrl = topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY);
+ Configurations configurations = SampleUtil.getSampleConfigs();
+ Map<String, Object> globalConfig = configurations.getGlobalConfig();
+ globalConfig.put("solr.zookeeper", solrComponent.getZookeeperUrl());
+ ConfigurationsUtils.writeGlobalConfigToZookeeper(JSONUtils.INSTANCE.toJSON(globalConfig), testZookeeperUrl);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+ })
+ .build();
+ return solrComponent;
+ }
+
+ @Override
+ public Processor<List<Map<String, Object>>> getProcessor(final List<byte[]> inputMessages) {
+ return new Processor<List<Map<String, Object>>>() {
+ List<Map<String, Object>> docs = null;
+ public ReadinessState process(ComponentRunner runner) {
+ SolrComponent solrComponent = runner.getComponent("search", SolrComponent.class);
+ if (solrComponent.hasCollection(collection)) {
+ List<Map<String, Object>> docsFromDisk;
+ try {
+ docs = solrComponent.getAllIndexedDocs(collection);
+ docsFromDisk = readDocsFromDisk(hdfsDir);
+ System.out.println(docs.size() + " vs " + inputMessages.size() + " vs " + docsFromDisk.size());
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to retrieve indexed documents.", e);
+ }
+ if (docs.size() < inputMessages.size() || docs.size() != docsFromDisk.size()) {
+ return ReadinessState.NOT_READY;
+ } else {
+ return ReadinessState.READY;
+ }
+ } else {
+ return ReadinessState.NOT_READY;
+ }
+ }
+
+ public List<Map<String, Object>> getResult() {
+ return docs;
+ }
+ };
+ }
+
+ @Override
+ public void setAdditionalProperties(Properties topologyProperties) {
+ topologyProperties.setProperty("writer.class.name", "org.apache.metron.solr.writer.SolrWriter");
+ }
+
+ @Override
+ public String cleanField(String field) {
+ return field.replaceFirst("_[dfils]$", "");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
index 0993e0d..91c34af 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
@@ -17,11 +17,9 @@
*/
package org.apache.metron.solr.writer;
-import backtype.storm.tuple.Tuple;
-import org.apache.metron.common.configuration.Configurations;
import org.apache.metron.common.configuration.EnrichmentConfigurations;
import org.apache.metron.common.configuration.writer.EnrichmentWriterConfiguration;
-import org.apache.metron.integration.utils.SampleUtil;
+import org.apache.metron.enrichment.integration.utils.SampleUtil;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.common.SolrInputDocument;
import org.hamcrest.Description;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml
new file mode 100644
index 0000000..d088c4b
--- /dev/null
+++ b/metron-platform/metron-writer/pom.xml
@@ -0,0 +1,231 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-platform</artifactId>
+ <version>0.2.0BETA</version>
+ </parent>
+ <artifactId>metron-writer</artifactId>
+ <name>metron-writer</name>
+ <description>Components common to all enrichments</description>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ <commons.config.version>1.10</commons.config.version>
+ <antlr.version>4.5</antlr.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${global_hbase_version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${global_hbase_version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${global_storm_version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.9.2</artifactId>
+ <version>${global_kafka_version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jmx</groupId>
+ <artifactId>jmxri</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jdmk</groupId>
+ <artifactId>jmxtools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jms</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-hdfs</artifactId>
+ <version>${global_storm_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${global_hadoop_version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ </exclusions>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ </dependencies>
+
+ <reporting>
+ <plugins>
+ <!-- Normally, dependency report takes time, skip it -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>2.7</version>
+
+ <configuration>
+ <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>emma-maven-plugin</artifactId>
+ <version>1.0-alpha-3</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-pmd-plugin</artifactId>
+ <configuration>
+ <targetJdk>${global_java_version}</targetJdk>
+ </configuration>
+ </plugin>
+ </plugins>
+ </reporting>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <source>${global_java_version}</source>
+ <compilerArgument>-Xlint:unchecked</compilerArgument>
+ <target>${global_java_version}</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr4-maven-plugin</artifactId>
+ <version>${antlr.version}</version>
+ <configuration>
+ <outputDirectory>${basedir}/src/main/java</outputDirectory>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>antlr4</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${global_shade_version}</version>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <artifactSet>
+ <excludes>
+ <exclude>*slf4j*</exclude>
+ </excludes>
+ </artifactSet>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>org.apache.metron.guava</shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resource>.yaml</resource>
+ </transformer>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass></mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/AbstractWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/AbstractWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/AbstractWriter.java
new file mode 100644
index 0000000..0fa596a
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/AbstractWriter.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer;
+
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+
+public abstract class AbstractWriter {
+ public AbstractWriter() {}
+ public abstract void configure(String sensorName, WriterConfiguration configuration);
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
new file mode 100644
index 0000000..ba647bd
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.tuple.Tuple;
+import com.google.common.collect.Iterables;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.utils.ErrorUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+public class BulkWriterComponent<MESSAGE_T> {
+ public static final Logger LOG = LoggerFactory
+ .getLogger(BulkWriterComponent.class);
+ private Map<String, Collection<Tuple>> sensorTupleMap = new HashMap<>();
+ private Map<String, List<MESSAGE_T>> sensorMessageMap = new HashMap<>();
+ private OutputCollector collector;
+ private boolean handleCommit = true;
+ private boolean handleError = true;
+ public BulkWriterComponent(OutputCollector collector) {
+ this.collector = collector;
+ }
+
+ public BulkWriterComponent(OutputCollector collector, boolean handleCommit, boolean handleError) {
+ this(collector);
+ this.handleCommit = handleCommit;
+ this.handleError = handleError;
+ }
+
+ public void commit(Iterable<Tuple> tuples) {
+ tuples.forEach(t -> collector.ack(t));
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Acking " + Iterables.size(tuples) + " tuples");
+ }
+ }
+
+ public void error(Throwable e, Iterable<Tuple> tuples) {
+ tuples.forEach(t -> collector.ack(t));
+ if(!Iterables.isEmpty(tuples)) {
+ LOG.error("Failing " + Iterables.size(tuples) + " tuples", e);
+ ErrorUtils.handleError(collector, e, Constants.ERROR_STREAM);
+ }
+ }
+
+ protected Collection<Tuple> createTupleCollection() {
+ return new ArrayList<>();
+ }
+
+
+ public void errorAll(Throwable e) {
+ for(Map.Entry<String, Collection<Tuple>> kv : sensorTupleMap.entrySet()) {
+ error(e, kv.getValue());
+ sensorTupleMap.remove(kv.getKey());
+ sensorMessageMap.remove(kv.getKey());
+ }
+ }
+
+ public void errorAll(String sensorType, Throwable e) {
+ error(e, Optional.ofNullable(sensorTupleMap.get(sensorType)).orElse(new ArrayList<>()));
+ sensorTupleMap.remove(sensorType);
+ sensorMessageMap.remove(sensorType);
+ }
+ public void write( String sensorType
+ , Tuple tuple
+ , MESSAGE_T message
+ , BulkMessageWriter<MESSAGE_T> bulkMessageWriter
+ , WriterConfiguration configurations
+ ) throws Exception
+ {
+ int batchSize = configurations.getBatchSize(sensorType);
+ Collection<Tuple> tupleList = sensorTupleMap.get(sensorType);
+ if (tupleList == null) {
+ tupleList = createTupleCollection();
+ }
+ tupleList.add(tuple);
+ List<MESSAGE_T> messageList = sensorMessageMap.get(sensorType);
+ if (messageList == null) {
+ messageList = new ArrayList<>();
+ }
+ messageList.add(message);
+
+ if (tupleList.size() < batchSize) {
+ sensorTupleMap.put(sensorType, tupleList);
+ sensorMessageMap.put(sensorType, messageList);
+ } else {
+ try {
+ bulkMessageWriter.write(sensorType, configurations, tupleList, messageList);
+ if(handleCommit) {
+ commit(tupleList);
+ }
+
+ } catch (Throwable e) {
+ if(handleError) {
+ error(e, tupleList);
+ }
+ else {
+ throw e;
+ }
+ }
+ finally {
+ sensorTupleMap.remove(sensorType);
+ sensorMessageMap.remove(sensorType);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java
new file mode 100644
index 0000000..b9aee7e
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer;
+
+import backtype.storm.tuple.Tuple;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.utils.ConversionUtils;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.function.Function;
+
+public class NoopWriter extends AbstractWriter implements BulkMessageWriter<Object> {
+
+ public static class RandomLatency implements Function<Void, Void> {
+ private int min;
+ private int max;
+
+ public RandomLatency(int min, int max) {
+ this.min = min;
+ this.max = max;
+ }
+
+ public int getMin() {
+ return min;
+ }
+ public int getMax() {
+ return max;
+ }
+
+ @Override
+ public Void apply(Void aVoid) {
+ int sleepMs = ThreadLocalRandom.current().nextInt(min, max + 1);
+ try {
+ Thread.sleep(sleepMs);
+ } catch (InterruptedException e) {
+ }
+ return null;
+ }
+ }
+
+ public static class FixedLatency implements Function<Void, Void> {
+ private int latency;
+ public FixedLatency(int latency) {
+ this.latency = latency;
+ }
+ public int getLatency() {
+ return latency;
+ }
+
+ @Override
+ public Void apply(Void aVoid) {
+ if(latency > 0) {
+ try {
+ Thread.sleep(latency);
+ } catch (InterruptedException e) {
+ }
+ }
+ return null;
+ }
+ }
+ Function<Void, Void> sleepFunction = null;
+
+ public NoopWriter withLatency(String sleepConfig) {
+ sleepFunction = getSleepFunction(sleepConfig);
+ return this;
+ }
+
+
+ private Function<Void, Void> getSleepFunction(String sleepConfig) {
+ String usageMessage = "Unexpected: " + sleepConfig + " Expected value: integer for a fixed sleep duration in milliseconds (e.g. 10) " +
+ "or a range of latencies separated by a comma (e.g. \"10, 20\") to sleep a random amount in that range.";
+ try {
+ if (sleepConfig.contains(",")) {
+ // random latency within a range.
+ Iterable<String> it = Splitter.on(',').split(sleepConfig);
+ Integer min = ConversionUtils.convert(Iterables.getFirst(it, "").trim(), Integer.class);
+ Integer max= ConversionUtils.convert(Iterables.getLast(it, "").trim(), Integer.class);
+ if (min != null && max != null) {
+ return new RandomLatency(min, max);
+ }
+ } else {
+ //fixed latency
+ Integer latency = ConversionUtils.convert(sleepConfig.trim(), Integer.class);
+ if(latency != null) {
+ return new FixedLatency(latency);
+ }
+ }
+ }
+ catch(Throwable t) {
+ throw new IllegalArgumentException(usageMessage, t);
+ }
+ throw new IllegalArgumentException(usageMessage);
+ }
+
+ @Override
+ public void configure(String sensorName, WriterConfiguration configuration) {
+ Map<String, Object> config = configuration.getSensorConfig(sensorName);
+ if(config != null) {
+ Object noopLatency = config.get("noopLatency");
+ if(noopLatency != null) {
+ sleepFunction = getSleepFunction(noopLatency.toString());
+ }
+ }
+ }
+
+ @Override
+ public void init(Map stormConf, WriterConfiguration config) throws Exception {
+ }
+
+ @Override
+ public void write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<Object> messages) throws Exception {
+ if(sleepFunction != null) {
+ sleepFunction.apply(null);
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java
new file mode 100644
index 0000000..3df3e9d
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer;
+
+import backtype.storm.tuple.Tuple;
+import com.google.common.collect.Iterables;
+import org.apache.metron.common.configuration.writer.SingleBatchConfigurationFacade;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.interfaces.MessageWriter;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+public class WriterToBulkWriter<MESSAGE_T> implements BulkMessageWriter<MESSAGE_T>, Serializable {
+ MessageWriter<MESSAGE_T> messageWriter;
+
+ public static transient Function<WriterConfiguration, WriterConfiguration> TRANSFORMATION = config -> new SingleBatchConfigurationFacade(config);
+
+ public WriterToBulkWriter(MessageWriter<MESSAGE_T> messageWriter) {
+ this.messageWriter = messageWriter;
+ }
+ @Override
+ public void init(Map stormConf, WriterConfiguration config) throws Exception {
+ messageWriter.init();
+ }
+
+ @Override
+ public void write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<MESSAGE_T> messages) throws Exception {
+ if(messages.size() > 1) {
+ throw new IllegalStateException("WriterToBulkWriter expects a batch of exactly 1");
+ }
+ messageWriter.write(sensorType, configurations, Iterables.getFirst(tuples, null), Iterables.getFirst(messages, null));
+ }
+
+ @Override
+ public void close() throws Exception {
+ messageWriter.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
new file mode 100644
index 0000000..91e8446
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer.bolt;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt;
+import org.apache.metron.common.configuration.writer.EnrichmentWriterConfiguration;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.MessageWriter;
+import org.apache.metron.common.utils.MessageUtils;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.writer.BulkWriterComponent;
+import org.apache.metron.writer.WriterToBulkWriter;
+import org.apache.metron.writer.message.MessageGetter;
+import org.apache.metron.writer.message.MessageGetters;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.function.Function;
+
+public class BulkMessageWriterBolt extends ConfiguredEnrichmentBolt {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(BulkMessageWriterBolt.class);
+ private BulkMessageWriter<JSONObject> bulkMessageWriter;
+ private BulkWriterComponent<JSONObject> writerComponent;
+ private String messageGetterStr = MessageGetters.NAMED.name();
+ private transient MessageGetter messageGetter = null;
+ private transient Function<WriterConfiguration, WriterConfiguration> configurationTransformation;
+ public BulkMessageWriterBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
+
+ public BulkMessageWriterBolt withBulkMessageWriter(BulkMessageWriter<JSONObject > bulkMessageWriter) {
+ this.bulkMessageWriter = bulkMessageWriter;
+ return this;
+ }
+
+ public BulkMessageWriterBolt withMessageWriter(MessageWriter<JSONObject> messageWriter) {
+ this.bulkMessageWriter = new WriterToBulkWriter<>(messageWriter);
+ return this;
+ }
+
+ public BulkMessageWriterBolt withMessageGetter(String messageGetter) {
+ this.messageGetterStr = messageGetter;
+ return this;
+ }
+
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ this.writerComponent = new BulkWriterComponent<>(collector);
+ super.prepare(stormConf, context, collector);
+ messageGetter = MessageGetters.valueOf(messageGetterStr);
+ if(bulkMessageWriter instanceof WriterToBulkWriter) {
+ configurationTransformation = WriterToBulkWriter.TRANSFORMATION;
+ }
+ else {
+ configurationTransformation = x -> x;
+ }
+ try {
+ bulkMessageWriter.init(stormConf
+ , configurationTransformation.apply(new EnrichmentWriterConfiguration(getConfigurations()))
+ );
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void execute(Tuple tuple) {
+ JSONObject message = messageGetter.getMessage(tuple);
+ String sensorType = MessageUtils.getSensorType(message);
+ try
+ {
+ writerComponent.write(sensorType
+ , tuple
+ , message
+ , bulkMessageWriter
+ , configurationTransformation.apply(new EnrichmentWriterConfiguration(getConfigurations()))
+ );
+ }
+ catch(Exception e) {
+ throw new RuntimeException("This should have been caught in the writerComponent. If you see this, file a JIRA", e);
+ }
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declareStream(Constants.ERROR_STREAM, new Fields("message"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
new file mode 100644
index 0000000..01f1245
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.writer.hdfs;
+
+import backtype.storm.tuple.Tuple;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.storm.hdfs.bolt.format.FileNameFormat;
+import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
+import org.apache.storm.hdfs.bolt.rotation.NoRotationPolicy;
+import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
+import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
+import org.apache.storm.hdfs.common.rotation.RotationAction;
+import org.json.simple.JSONObject;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class HdfsWriter implements BulkMessageWriter<JSONObject>, Serializable {
+ List<RotationAction> rotationActions = new ArrayList<>();
+ FileRotationPolicy rotationPolicy = new NoRotationPolicy();
+ SyncPolicy syncPolicy = new CountSyncPolicy(1); //sync every time, duh.
+ FileNameFormat fileNameFormat;
+ Map<String, SourceHandler> sourceHandlerMap = new HashMap<>();
+ transient Map stormConfig;
+ public HdfsWriter withFileNameFormat(FileNameFormat fileNameFormat){
+ this.fileNameFormat = fileNameFormat;
+ return this;
+ }
+
+ public HdfsWriter withSyncPolicy(SyncPolicy syncPolicy){
+ this.syncPolicy = syncPolicy;
+ return this;
+ }
+ public HdfsWriter withRotationPolicy(FileRotationPolicy rotationPolicy){
+ this.rotationPolicy = rotationPolicy;
+ return this;
+ }
+
+ public HdfsWriter addRotationAction(RotationAction action){
+ this.rotationActions.add(action);
+ return this;
+ }
+
+ @Override
+ public void init(Map stormConfig, WriterConfiguration configurations) {
+ this.stormConfig = stormConfig;
+ }
+
+
+ @Override
+ public void write(String sourceType
+ , WriterConfiguration configurations
+ , Iterable<Tuple> tuples
+ , List<JSONObject> messages
+ ) throws Exception
+ {
+ SourceHandler handler = getSourceHandler(sourceType);
+ handler.handle(messages);
+ }
+
+ @Override
+ public void close() {
+ for(SourceHandler handler : sourceHandlerMap.values()) {
+ handler.close();
+ }
+ }
+ private synchronized SourceHandler getSourceHandler(String sourceType) throws IOException {
+ SourceHandler ret = sourceHandlerMap.get(sourceType);
+ if(ret == null) {
+ ret = new SourceHandler(rotationActions, rotationPolicy, syncPolicy, new SourceFileNameFormat(sourceType, fileNameFormat), stormConfig);
+ sourceHandlerMap.put(sourceType, ret);
+ }
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
new file mode 100644
index 0000000..1c345b4
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.hdfs;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+import org.apache.storm.hdfs.common.rotation.RotationAction;
+
+import java.io.IOException;
+
+public class SourceAwareMoveAction implements RotationAction{
+ private static final Logger LOG = Logger.getLogger(SourceHandler.class);
+ private String destination;
+
+ public SourceAwareMoveAction toDestination(String destDir){
+ destination = destDir;
+ return this;
+ }
+
+ private static String getSource(Path filePath) {
+ return filePath.getParent().getName();
+ }
+
+ @Override
+ public void execute(FileSystem fileSystem, Path filePath) throws IOException {
+ Path destPath = new Path(new Path(destination, getSource(filePath)), filePath.getName());
+ LOG.info("Moving file " + filePath + " to " + destPath);
+ boolean success = fileSystem.rename(filePath, destPath);
+ return;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java
new file mode 100644
index 0000000..ae0242d
--- /dev/null
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.writer.hdfs;
+
+import backtype.storm.task.TopologyContext;
+import org.apache.storm.hdfs.bolt.format.FileNameFormat;
+
+import java.util.Map;
+
+public class SourceFileNameFormat implements FileNameFormat {
+ FileNameFormat delegate;
+ String sourceType;
+ public SourceFileNameFormat(String sourceType, FileNameFormat delegate) {
+ this.delegate = delegate;
+ this.sourceType = sourceType;
+ }
+
+ @Override
+ public void prepare(Map map, TopologyContext topologyContext) {
+ this.delegate.prepare(map, topologyContext);
+ }
+
+ @Override
+ public String getName(long l, long l1) {
+ return delegate.getName(l, l1);
+ }
+
+ @Override
+ public String getPath() {
+ return delegate.getPath() + "/" + sourceType;
+ }
+}
[4/5] incubator-metron git commit: METRON-154: Decouple enrichment
and indexing closes apache/incubator-metron#192
Posted by ce...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml
index 226b686..0f391c4 100644
--- a/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml
+++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/test.yaml
@@ -21,7 +21,7 @@ config:
components:
# Enrichment
- id: "geoEnrichmentAdapter"
- className: "org.apache.metron.integration.mock.MockGeoAdapter"
+ className: "org.apache.metron.enrichment.integration.mock.MockGeoAdapter"
- id: "geoEnrichment"
className: "org.apache.metron.enrichment.configuration.Enrichment"
constructorArgs:
@@ -111,27 +111,17 @@ components:
args:
- ref: "simpleHBaseThreatIntelEnrichment"
- - id: "fileNameFormat"
- className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat"
- configMethods:
- - name: "withPrefix"
- args:
- - "enrichment-"
- - name: "withExtension"
- args:
- - ".json"
- - name: "withPath"
- args:
- - "${index.hdfs.output}"
+
#indexing
- - id: "hdfsWriter"
- className: "org.apache.metron.writer.hdfs.HdfsWriter"
+ - id: "kafkaWriter"
+ className: "org.apache.metron.writer.kafka.KafkaWriter"
configMethods:
- - name: "withFileNameFormat"
+ - name: "withTopic"
args:
- - ref: "fileNameFormat"
- - id: "indexWriter"
- className: "${writer.class.name}"
+ - "${enrichment.output.topic}"
+ - name: "withZkQuorum"
+ args:
+ - "${kafka.zk}"
#kafka/zookeeper
- id: "zkHosts"
@@ -249,22 +239,14 @@ bolts:
- name: "withMaxTimeRetain"
args: [10]
# Indexing Bolts
- - id: "indexingBolt"
- className: "org.apache.metron.enrichment.bolt.BulkMessageWriterBolt"
- constructorArgs:
- - "${kafka.zk}"
- configMethods:
- - name: "withBulkMessageWriter"
- args:
- - ref: "indexWriter"
- - id: "hdfsIndexingBolt"
- className: "org.apache.metron.enrichment.bolt.BulkMessageWriterBolt"
+ - id: "outputBolt"
+ className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
constructorArgs:
- "${kafka.zk}"
configMethods:
- - name: "withBulkMessageWriter"
+ - name: "withMessageWriter"
args:
- - ref: "hdfsWriter"
+ - ref: "kafkaWriter"
streams:
@@ -361,24 +343,12 @@ streams:
streamId: "message"
type: FIELDS
args: ["key"]
-#indexing
- - name: "threatIntelJoin -> indexing"
+
+#output
+ - name: "threatIntelJoin -> output"
from: "threatIntelJoinBolt"
- to: "indexingBolt"
+ to: "outputBolt"
grouping:
streamId: "message"
type: FIELDS
args: ["key"]
- - name: "threatIntelJoin -> hdfs"
- from: "threatIntelJoinBolt"
- to: "hdfsIndexingBolt"
- grouping:
- streamId: "message"
- type: SHUFFLE
-
- - name: "indexingBolt -> errorIndexingBolt"
- from: "indexingBolt"
- to: "indexingBolt"
- grouping:
- streamId: "error"
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
deleted file mode 100644
index 772dfc6..0000000
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBolt.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.enrichment.bolt;
-
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Fields;
-import backtype.storm.tuple.Tuple;
-import org.apache.metron.common.Constants;
-import org.apache.metron.common.bolt.ConfiguredEnrichmentBolt;
-import org.apache.metron.common.configuration.writer.EnrichmentWriterConfiguration;
-import org.apache.metron.common.utils.ErrorUtils;
-import org.apache.metron.common.utils.MessageUtils;
-import org.apache.metron.common.interfaces.BulkMessageWriter;
-import org.apache.metron.common.writer.BulkWriterComponent;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-
-public class BulkMessageWriterBolt extends ConfiguredEnrichmentBolt {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(BulkMessageWriterBolt.class);
- private BulkMessageWriter<JSONObject> bulkMessageWriter;
- private BulkWriterComponent<JSONObject> writerComponent;
- public BulkMessageWriterBolt(String zookeeperUrl) {
- super(zookeeperUrl);
- }
-
- public BulkMessageWriterBolt withBulkMessageWriter(BulkMessageWriter<JSONObject > bulkMessageWriter) {
- this.bulkMessageWriter = bulkMessageWriter;
- return this;
- }
-
- @Override
- public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
- this.writerComponent = new BulkWriterComponent<>(collector);
- super.prepare(stormConf, context, collector);
- try {
- bulkMessageWriter.init(stormConf, new EnrichmentWriterConfiguration(getConfigurations()));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
-
- @SuppressWarnings("unchecked")
- @Override
- public void execute(Tuple tuple) {
- JSONObject message =(JSONObject)tuple.getValueByField("message");
- String sensorType = MessageUtils.getSensorType(message);
- try
- {
- writerComponent.write(sensorType, tuple, message, bulkMessageWriter, new EnrichmentWriterConfiguration(getConfigurations()));
- }
- catch(Exception e) {
- throw new RuntimeException("This should have been caught in the writerComponent. If you see this, file a JIRA", e);
- }
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declareStream(Constants.ERROR_STREAM, new Fields("message"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java
index 3c38824..3b7e437 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/lookup/accesstracker/BloomAccessTracker.java
@@ -17,13 +17,12 @@
*/
package org.apache.metron.enrichment.lookup.accesstracker;
-import com.google.common.hash.BloomFilter;
-import com.google.common.hash.Funnel;
-import com.google.common.hash.PrimitiveSink;
+import org.apache.metron.common.utils.BloomFilter;
import org.apache.metron.enrichment.lookup.LookupKey;
import java.io.*;
import java.util.Map;
+import java.util.function.Function;
public class BloomAccessTracker implements AccessTracker {
private static final long serialVersionUID = 1L;
@@ -31,25 +30,14 @@ public class BloomAccessTracker implements AccessTracker {
public static final String FALSE_POSITIVE_RATE_KEY = "false_positive_rate";
public static final String NAME_KEY = "name";
- private static class LookupKeyFunnel implements Funnel<LookupKey> {
- @Override
- public void funnel(LookupKey lookupKey, PrimitiveSink primitiveSink) {
- primitiveSink.putBytes(lookupKey.toBytes());
- }
-
- @Override
- public boolean equals(Object obj) {
- return this.getClass().equals(obj.getClass());
- }
+ public static class LookupKeySerializer implements Function<LookupKey, byte[]>, Serializable {
@Override
- public int hashCode() {
- return super.hashCode() * 31;
+ public byte[] apply(LookupKey lookupKey) {
+ return lookupKey.toBytes();
}
}
- private static Funnel<LookupKey> LOOKUPKEY_FUNNEL = new LookupKeyFunnel();
-
BloomFilter<LookupKey> filter;
String name;
int expectedInsertions;
@@ -60,7 +48,7 @@ public class BloomAccessTracker implements AccessTracker {
this.name = name;
this.expectedInsertions = expectedInsertions;
this.falsePositiveRate = falsePositiveRate;
- filter = BloomFilter.create(LOOKUPKEY_FUNNEL, expectedInsertions, falsePositiveRate);
+ filter = new BloomFilter<LookupKey>(new LookupKeySerializer(), expectedInsertions, falsePositiveRate);
}
public BloomAccessTracker() {}
public BloomAccessTracker(Map<String, Object> config) {
@@ -73,7 +61,7 @@ public class BloomAccessTracker implements AccessTracker {
@Override
public void logAccess(LookupKey key) {
numInsertions++;
- filter.put(key);
+ filter.add(key);
}
@Override
@@ -81,7 +69,7 @@ public class BloomAccessTracker implements AccessTracker {
expectedInsertions = toInt(config.get(EXPECTED_INSERTIONS_KEY));
falsePositiveRate = toDouble(config.get(FALSE_POSITIVE_RATE_KEY));
name = config.get(NAME_KEY).toString();
- filter = BloomFilter.create(LOOKUPKEY_FUNNEL, expectedInsertions, falsePositiveRate);
+ filter = new BloomFilter<LookupKey>(new LookupKeySerializer(), expectedInsertions, falsePositiveRate);
}
@Override
@@ -91,7 +79,7 @@ public class BloomAccessTracker implements AccessTracker {
@Override
public void reset() {
- filter = BloomFilter.create(LOOKUPKEY_FUNNEL, expectedInsertions, falsePositiveRate);
+ filter = new BloomFilter<LookupKey>(new LookupKeySerializer(), expectedInsertions, falsePositiveRate);
}
private static double toDouble(Object o) {
@@ -129,7 +117,7 @@ public class BloomAccessTracker implements AccessTracker {
throw new IllegalStateException("Unable to union access tracker, because this tracker is not initialized.");
}
if(tracker instanceof BloomAccessTracker ) {
- filter.putAll(((BloomAccessTracker)tracker).getFilter());
+ filter.merge(((BloomAccessTracker)tracker).getFilter());
return this;
}
else {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java
new file mode 100644
index 0000000..cfdae33
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/writer/SimpleHbaseEnrichmentWriter.java
@@ -0,0 +1,275 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.enrichment.writer;
+
+import backtype.storm.tuple.Tuple;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.common.utils.ReflectionUtils;
+import org.apache.metron.enrichment.converter.EnrichmentConverter;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.hbase.HTableProvider;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.writer.AbstractWriter;
+import org.json.simple.JSONObject;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class SimpleHbaseEnrichmentWriter extends AbstractWriter implements BulkMessageWriter<JSONObject>, Serializable {
+ public enum Configurations {
+ HBASE_TABLE("shew.table")
+ ,HBASE_CF("shew.cf")
+ ,KEY_COLUMNS("shew.keyColumns")
+ ,KEY_DELIM("shew.keyDelim")
+ ,ENRICHMENT_TYPE("shew.enrichmentType")
+ ,VALUE_COLUMNS("shew.valueColumns")
+ ,HBASE_PROVIDER("shew.hbaseProvider")
+ ;
+ String key;
+ Configurations(String key) {
+ this.key = key;
+ }
+ public String getKey() {
+ return key;
+ }
+ public Object get(Map<String, Object> config) {
+ return config.get(key);
+ }
+ public <T> T getAndConvert(Map<String, Object> config, Class<T> clazz) {
+ Object o = get(config);
+ if(o != null) {
+ return ConversionUtils.convert(o, clazz);
+ }
+ return null;
+ }
+ }
+ public static class KeyTransformer {
+ List<String> keys = new ArrayList<>();
+ Set<String> keySet;
+ private String delim = ":";
+ public KeyTransformer(String key) {
+ this(key, null);
+ }
+ public KeyTransformer(String key, String delim) {
+ keys.add(key);
+ keySet = new HashSet<>(this.keys);
+ this.delim = delim == null?this.delim:delim;
+ }
+ public KeyTransformer(Iterable<String> keys) {
+ this(keys, null);
+ }
+ public KeyTransformer(Iterable<String> keys, String delim) {
+ Iterables.addAll(this.keys, keys);
+ keySet = new HashSet<>(this.keys);
+ this.delim = delim == null?this.delim:delim;
+ }
+
+ public String transform(final JSONObject message) {
+ return
+ keys.stream().map( x -> {
+ Object o = message.get(x);
+ return o == null?"":o.toString();
+ }).collect(Collectors.joining(delim));
+ }
+ }
+ private transient EnrichmentConverter converter;
+ private String tableName;
+ private String cf;
+ private HTableInterface table;
+ private TableProvider provider;
+ private Map.Entry<Object, KeyTransformer> keyTransformer;
+
+ public SimpleHbaseEnrichmentWriter() {
+ }
+
+ @Override
+ public void configure(String sensorName, WriterConfiguration configuration) {
+ String hbaseProviderImpl = Configurations.HBASE_PROVIDER.getAndConvert(configuration.getSensorConfig(sensorName),String.class);
+ if(hbaseProviderImpl != null) {
+ provider = ReflectionUtils.createInstance(hbaseProviderImpl);
+ }
+ if(converter == null) {
+ converter = new EnrichmentConverter();
+ }
+ }
+
+ @Override
+ public void init(Map stormConf, WriterConfiguration configuration) throws Exception {
+ if(converter == null) {
+ converter = new EnrichmentConverter();
+ }
+ }
+
+ protected synchronized TableProvider getProvider() {
+ if(provider == null) {
+
+ provider = new HTableProvider();
+ }
+ return provider;
+ }
+
+ public HTableInterface getTable(String tableName, String cf) throws IOException {
+ synchronized(this) {
+ boolean isInitial = this.tableName == null || this.cf == null;
+ boolean isValid = tableName != null && cf != null;
+
+ if( isInitial || (isValid && (!this.tableName.equals(tableName) || !this.cf.equals(cf)) )
+ )
+ {
+ Configuration conf = HBaseConfiguration.create();
+ //new table connection
+ if(table != null) {
+ table.close();
+ }
+ table = getProvider().getTable(conf, tableName);
+ this.tableName = tableName;
+ this.cf = cf;
+ }
+ return table;
+ }
+ }
+
+ public HTableInterface getTable(Map<String, Object> config) throws IOException {
+ return getTable(Configurations.HBASE_TABLE.getAndConvert(config, String.class)
+ ,Configurations.HBASE_CF.getAndConvert(config, String.class)
+ );
+
+ }
+
+
+ private List<String> getColumns(Object keyColumnsObj, boolean allowNull) {
+ Object o = keyColumnsObj;
+ if(allowNull && keyColumnsObj == null) {
+ return Collections.emptyList();
+ }
+ if(o instanceof String) {
+ return ImmutableList.of(o.toString());
+ }
+ else if (o instanceof List) {
+ List<String> keyCols = new ArrayList<>();
+ for(Object key : (List)o) {
+ keyCols.add(key.toString());
+ }
+ return keyCols;
+ }
+ else {
+ throw new RuntimeException("Unable to get columns: " + o);
+ }
+ }
+
+ private KeyTransformer getTransformer(Map<String, Object> config) {
+ Object o = Configurations.KEY_COLUMNS.get(config);
+ KeyTransformer transformer = null;
+ if(keyTransformer != null && keyTransformer.getKey() == o) {
+ return keyTransformer.getValue();
+ }
+ else {
+ List<String> keys = getColumns(o, false);
+ Object delimObj = Configurations.KEY_DELIM.get(config);
+ String delim = (delimObj == null || !(delimObj instanceof String))?null:delimObj.toString();
+ transformer = new KeyTransformer(keys, delim);
+ keyTransformer = new AbstractMap.SimpleEntry<>(o, transformer);
+ return transformer;
+ }
+ }
+
+
+ private EnrichmentValue getValue( JSONObject message
+ , Set<String> keyColumns
+ , Set<String> valueColumns
+ )
+ {
+ Map<String, Object> metadata = new HashMap<>();
+ if(valueColumns == null || valueColumns.isEmpty()) {
+ for (Object kv : message.entrySet()) {
+ Map.Entry<Object, Object> entry = (Map.Entry<Object, Object>) kv;
+ if (!keyColumns.contains(entry.getKey())) {
+ metadata.put(entry.getKey().toString(), entry.getValue());
+ }
+ }
+ return new EnrichmentValue(metadata);
+ }
+ else {
+ for (Object kv : message.entrySet()) {
+ Map.Entry<Object, Object> entry = (Map.Entry<Object, Object>) kv;
+ if (valueColumns.contains(entry.getKey())) {
+ metadata.put(entry.getKey().toString(), entry.getValue());
+ }
+ }
+ return new EnrichmentValue(metadata);
+ }
+ }
+
+ private EnrichmentKey getKey(JSONObject message, KeyTransformer transformer, String enrichmentType) {
+ if(enrichmentType != null) {
+ return new EnrichmentKey(enrichmentType, transformer.transform(message));
+ }
+ else {
+ return null;
+ }
+ }
+
+ @Override
+ public void write( String sensorType
+ , WriterConfiguration configurations
+ , Iterable<Tuple> tuples
+ , List<JSONObject> messages
+ ) throws Exception
+ {
+ Map<String, Object> sensorConfig = configurations.getSensorConfig(sensorType);
+ HTableInterface table = getTable(sensorConfig);
+ KeyTransformer transformer = getTransformer(sensorConfig);
+ Object enrichmentTypeObj = Configurations.ENRICHMENT_TYPE.get(sensorConfig);
+ String enrichmentType = enrichmentTypeObj == null?null:enrichmentTypeObj.toString();
+ Set<String> valueColumns = new HashSet<>(getColumns(Configurations.VALUE_COLUMNS.get(sensorConfig), true));
+ List<Put> puts = new ArrayList<>();
+ for(JSONObject message : messages) {
+ EnrichmentKey key = getKey(message, transformer, enrichmentType);
+ EnrichmentValue value = getValue(message, transformer.keySet, valueColumns);
+ if(key == null || value == null) {
+ continue;
+ }
+ Put put = converter.toPut(this.cf, key, value);
+ if(put != null) {
+ puts.add(put);
+ }
+ }
+ table.put(puts);
+ }
+
+ @Override
+ public void close() throws Exception {
+ synchronized(this) {
+ if(table != null) {
+ table.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
deleted file mode 100644
index f7e7aeb..0000000
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
+++ /dev/null
@@ -1,282 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.writer.hbase;
-
-import backtype.storm.tuple.Tuple;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.metron.common.configuration.Configurations;
-import org.apache.metron.common.configuration.ParserConfigurations;
-import org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.interfaces.BulkMessageWriter;
-import org.apache.metron.common.utils.ConversionUtils;
-import org.apache.metron.common.utils.ReflectionUtils;
-import org.apache.metron.common.writer.AbstractWriter;
-import org.apache.metron.enrichment.converter.EnrichmentConverter;
-import org.apache.metron.enrichment.converter.EnrichmentKey;
-import org.apache.metron.enrichment.converter.EnrichmentValue;
-import org.apache.metron.enrichment.converter.HbaseConverter;
-import org.apache.metron.hbase.HTableProvider;
-import org.apache.metron.hbase.TableProvider;
-import org.json.simple.JSONObject;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.*;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-public class SimpleHbaseEnrichmentWriter extends AbstractWriter implements BulkMessageWriter<JSONObject>, Serializable {
- public enum Configurations {
- HBASE_TABLE("shew.table")
- ,HBASE_CF("shew.cf")
- ,KEY_COLUMNS("shew.keyColumns")
- ,KEY_DELIM("shew.keyDelim")
- ,ENRICHMENT_TYPE("shew.enrichmentType")
- ,VALUE_COLUMNS("shew.valueColumns")
- ,HBASE_PROVIDER("shew.hbaseProvider")
- ;
- String key;
- Configurations(String key) {
- this.key = key;
- }
- public String getKey() {
- return key;
- }
- public Object get(Map<String, Object> config) {
- return config.get(key);
- }
- public <T> T getAndConvert(Map<String, Object> config, Class<T> clazz) {
- Object o = get(config);
- if(o != null) {
- return ConversionUtils.convert(o, clazz);
- }
- return null;
- }
- }
- public static class KeyTransformer {
- List<String> keys = new ArrayList<>();
- Set<String> keySet;
- private String delim = ":";
- public KeyTransformer(String key) {
- this(key, null);
- }
- public KeyTransformer(String key, String delim) {
- keys.add(key);
- keySet = new HashSet<>(this.keys);
- this.delim = delim == null?this.delim:delim;
- }
- public KeyTransformer(Iterable<String> keys) {
- this(keys, null);
- }
- public KeyTransformer(Iterable<String> keys, String delim) {
- Iterables.addAll(this.keys, keys);
- keySet = new HashSet<>(this.keys);
- this.delim = delim == null?this.delim:delim;
- }
-
- public String transform(final JSONObject message) {
- return
- keys.stream().map( x -> {
- Object o = message.get(x);
- return o == null?"":o.toString();
- }).collect(Collectors.joining(delim));
- }
- }
- private transient EnrichmentConverter converter;
- private String tableName;
- private String cf;
- private HTableInterface table;
- private TableProvider provider;
- private Map.Entry<Object, KeyTransformer> keyTransformer;
-
- public SimpleHbaseEnrichmentWriter() {
- }
-
- @Override
- public void configure(String sensorName, WriterConfiguration configuration) {
- String hbaseProviderImpl = Configurations.HBASE_PROVIDER.getAndConvert(configuration.getSensorConfig(sensorName),String.class);
- if(hbaseProviderImpl != null) {
- provider = ReflectionUtils.createInstance(hbaseProviderImpl);
- }
- if(converter == null) {
- converter = new EnrichmentConverter();
- }
- }
-
- @Override
- public void init(Map stormConf, WriterConfiguration configuration) throws Exception {
- if(converter == null) {
- converter = new EnrichmentConverter();
- }
- }
-
- protected synchronized TableProvider getProvider() {
- if(provider == null) {
-
- provider = new HTableProvider();
- }
- return provider;
- }
-
- public HTableInterface getTable(String tableName, String cf) throws IOException {
- synchronized(this) {
- boolean isInitial = this.tableName == null || this.cf == null;
- boolean isValid = tableName != null && cf != null;
-
- if( isInitial || (isValid && (!this.tableName.equals(tableName) || !this.cf.equals(cf)) )
- )
- {
- Configuration conf = HBaseConfiguration.create();
- //new table connection
- if(table != null) {
- table.close();
- }
- table = getProvider().getTable(conf, tableName);
- this.tableName = tableName;
- this.cf = cf;
- }
- return table;
- }
- }
-
- public HTableInterface getTable(Map<String, Object> config) throws IOException {
- return getTable(Configurations.HBASE_TABLE.getAndConvert(config, String.class)
- ,Configurations.HBASE_CF.getAndConvert(config, String.class)
- );
-
- }
-
-
- private List<String> getColumns(Object keyColumnsObj, boolean allowNull) {
- Object o = keyColumnsObj;
- if(allowNull && keyColumnsObj == null) {
- return Collections.emptyList();
- }
- if(o instanceof String) {
- return ImmutableList.of(o.toString());
- }
- else if (o instanceof List) {
- List<String> keyCols = new ArrayList<>();
- for(Object key : (List)o) {
- keyCols.add(key.toString());
- }
- return keyCols;
- }
- else {
- throw new RuntimeException("Unable to get columns: " + o);
- }
- }
-
- private KeyTransformer getTransformer(Map<String, Object> config) {
- Object o = Configurations.KEY_COLUMNS.get(config);
- KeyTransformer transformer = null;
- if(keyTransformer != null && keyTransformer.getKey() == o) {
- return keyTransformer.getValue();
- }
- else {
- List<String> keys = getColumns(o, false);
- Object delimObj = Configurations.KEY_DELIM.get(config);
- String delim = (delimObj == null || !(delimObj instanceof String))?null:delimObj.toString();
- transformer = new KeyTransformer(keys, delim);
- keyTransformer = new AbstractMap.SimpleEntry<>(o, transformer);
- return transformer;
- }
- }
-
-
- private EnrichmentValue getValue( JSONObject message
- , Set<String> keyColumns
- , Set<String> valueColumns
- )
- {
- Map<String, Object> metadata = new HashMap<>();
- if(valueColumns == null || valueColumns.isEmpty()) {
- for (Object kv : message.entrySet()) {
- Map.Entry<Object, Object> entry = (Map.Entry<Object, Object>) kv;
- if (!keyColumns.contains(entry.getKey())) {
- metadata.put(entry.getKey().toString(), entry.getValue());
- }
- }
- return new EnrichmentValue(metadata);
- }
- else {
- for (Object kv : message.entrySet()) {
- Map.Entry<Object, Object> entry = (Map.Entry<Object, Object>) kv;
- if (valueColumns.contains(entry.getKey())) {
- metadata.put(entry.getKey().toString(), entry.getValue());
- }
- }
- return new EnrichmentValue(metadata);
- }
- }
-
- private EnrichmentKey getKey(JSONObject message, KeyTransformer transformer, String enrichmentType) {
- if(enrichmentType != null) {
- return new EnrichmentKey(enrichmentType, transformer.transform(message));
- }
- else {
- return null;
- }
- }
-
- @Override
- public void write( String sensorType
- , WriterConfiguration configurations
- , Iterable<Tuple> tuples
- , List<JSONObject> messages
- ) throws Exception
- {
- Map<String, Object> sensorConfig = configurations.getSensorConfig(sensorType);
- HTableInterface table = getTable(sensorConfig);
- KeyTransformer transformer = getTransformer(sensorConfig);
- Object enrichmentTypeObj = Configurations.ENRICHMENT_TYPE.get(sensorConfig);
- String enrichmentType = enrichmentTypeObj == null?null:enrichmentTypeObj.toString();
- Set<String> valueColumns = new HashSet<>(getColumns(Configurations.VALUE_COLUMNS.get(sensorConfig), true));
- List<Put> puts = new ArrayList<>();
- for(JSONObject message : messages) {
- EnrichmentKey key = getKey(message, transformer, enrichmentType);
- EnrichmentValue value = getValue(message, transformer.keySet, valueColumns);
- if(key == null || value == null) {
- continue;
- }
- Put put = converter.toPut(this.cf, key, value);
- if(put != null) {
- puts.add(put);
- }
- }
- table.put(puts);
- }
-
- @Override
- public void close() throws Exception {
- synchronized(this) {
- if(table != null) {
- table.close();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
deleted file mode 100644
index 01f1245..0000000
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.writer.hdfs;
-
-import backtype.storm.tuple.Tuple;
-import org.apache.metron.common.configuration.EnrichmentConfigurations;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.interfaces.BulkMessageWriter;
-import org.apache.storm.hdfs.bolt.format.FileNameFormat;
-import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
-import org.apache.storm.hdfs.bolt.rotation.NoRotationPolicy;
-import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
-import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
-import org.apache.storm.hdfs.common.rotation.RotationAction;
-import org.json.simple.JSONObject;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class HdfsWriter implements BulkMessageWriter<JSONObject>, Serializable {
- List<RotationAction> rotationActions = new ArrayList<>();
- FileRotationPolicy rotationPolicy = new NoRotationPolicy();
- SyncPolicy syncPolicy = new CountSyncPolicy(1); //sync every time, duh.
- FileNameFormat fileNameFormat;
- Map<String, SourceHandler> sourceHandlerMap = new HashMap<>();
- transient Map stormConfig;
- public HdfsWriter withFileNameFormat(FileNameFormat fileNameFormat){
- this.fileNameFormat = fileNameFormat;
- return this;
- }
-
- public HdfsWriter withSyncPolicy(SyncPolicy syncPolicy){
- this.syncPolicy = syncPolicy;
- return this;
- }
- public HdfsWriter withRotationPolicy(FileRotationPolicy rotationPolicy){
- this.rotationPolicy = rotationPolicy;
- return this;
- }
-
- public HdfsWriter addRotationAction(RotationAction action){
- this.rotationActions.add(action);
- return this;
- }
-
- @Override
- public void init(Map stormConfig, WriterConfiguration configurations) {
- this.stormConfig = stormConfig;
- }
-
-
- @Override
- public void write(String sourceType
- , WriterConfiguration configurations
- , Iterable<Tuple> tuples
- , List<JSONObject> messages
- ) throws Exception
- {
- SourceHandler handler = getSourceHandler(sourceType);
- handler.handle(messages);
- }
-
- @Override
- public void close() {
- for(SourceHandler handler : sourceHandlerMap.values()) {
- handler.close();
- }
- }
- private synchronized SourceHandler getSourceHandler(String sourceType) throws IOException {
- SourceHandler ret = sourceHandlerMap.get(sourceType);
- if(ret == null) {
- ret = new SourceHandler(rotationActions, rotationPolicy, syncPolicy, new SourceFileNameFormat(sourceType, fileNameFormat), stormConfig);
- sourceHandlerMap.put(sourceType, ret);
- }
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
deleted file mode 100644
index 1c345b4..0000000
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.writer.hdfs;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Logger;
-import org.apache.storm.hdfs.common.rotation.RotationAction;
-
-import java.io.IOException;
-
-public class SourceAwareMoveAction implements RotationAction{
- private static final Logger LOG = Logger.getLogger(SourceHandler.class);
- private String destination;
-
- public SourceAwareMoveAction toDestination(String destDir){
- destination = destDir;
- return this;
- }
-
- private static String getSource(Path filePath) {
- return filePath.getParent().getName();
- }
-
- @Override
- public void execute(FileSystem fileSystem, Path filePath) throws IOException {
- Path destPath = new Path(new Path(destination, getSource(filePath)), filePath.getName());
- LOG.info("Moving file " + filePath + " to " + destPath);
- boolean success = fileSystem.rename(filePath, destPath);
- return;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java
deleted file mode 100644
index ae0242d..0000000
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceFileNameFormat.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.writer.hdfs;
-
-import backtype.storm.task.TopologyContext;
-import org.apache.storm.hdfs.bolt.format.FileNameFormat;
-
-import java.util.Map;
-
-public class SourceFileNameFormat implements FileNameFormat {
- FileNameFormat delegate;
- String sourceType;
- public SourceFileNameFormat(String sourceType, FileNameFormat delegate) {
- this.delegate = delegate;
- this.sourceType = sourceType;
- }
-
- @Override
- public void prepare(Map map, TopologyContext topologyContext) {
- this.delegate.prepare(map, topologyContext);
- }
-
- @Override
- public String getName(long l, long l1) {
- return delegate.getName(l, l1);
- }
-
- @Override
- public String getPath() {
- return delegate.getPath() + "/" + sourceType;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
deleted file mode 100644
index 0225137..0000000
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.writer.hdfs;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
-import org.apache.hadoop.hdfs.util.MD5FileUtils;
-import org.apache.hadoop.io.MD5Hash;
-import org.apache.log4j.Logger;
-import org.apache.storm.hdfs.bolt.format.FileNameFormat;
-import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
-import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy;
-import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
-import org.apache.storm.hdfs.common.rotation.RotationAction;
-import org.apache.storm.hdfs.common.security.HdfsSecurityUtil;
-import org.json.simple.JSONObject;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.*;
-
-public class SourceHandler {
- private static final Logger LOG = Logger.getLogger(SourceHandler.class);
- List<RotationAction> rotationActions = new ArrayList<>();
- FileRotationPolicy rotationPolicy;
- SyncPolicy syncPolicy;
- FileNameFormat fileNameFormat;
- private long offset = 0;
- private int rotation = 0;
- private transient FSDataOutputStream out;
- private transient Object writeLock;
- protected transient Timer rotationTimer; // only used for TimedRotationPolicy
- protected transient FileSystem fs;
- protected transient Path currentFile;
- public SourceHandler(List<RotationAction> rotationActions
- , FileRotationPolicy rotationPolicy
- , SyncPolicy syncPolicy
- , FileNameFormat fileNameFormat
- , Map config
- ) throws IOException {
- this.rotationActions = rotationActions;
- this.rotationPolicy = rotationPolicy;
- this.syncPolicy = syncPolicy;
- this.fileNameFormat = fileNameFormat;
- initialize(config);
- }
-
- public void handle(List<JSONObject> messages) throws Exception{
-
- for(JSONObject message : messages) {
- byte[] bytes = (message.toJSONString() + "\n").getBytes();
- synchronized (this.writeLock) {
- out.write(bytes);
- this.offset += bytes.length;
-
- if (this.syncPolicy.mark(null, this.offset)) {
- if (this.out instanceof HdfsDataOutputStream) {
- ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
- } else {
- this.out.hsync();
- }
- this.syncPolicy.reset();
- }
- }
-
- if (this.rotationPolicy.mark(null, this.offset)) {
- rotateOutputFile(); // synchronized
- this.offset = 0;
- this.rotationPolicy.reset();
- }
- }
- }
-
- private void initialize(Map config) throws IOException {
- this.writeLock = new Object();
- Configuration hdfsConfig = new Configuration();
- this.fs = FileSystem.get(new Configuration());
- HdfsSecurityUtil.login(config, hdfsConfig);
- this.currentFile = createOutputFile();
- if(this.rotationPolicy instanceof TimedRotationPolicy){
- long interval = ((TimedRotationPolicy)this.rotationPolicy).getInterval();
- this.rotationTimer = new Timer(true);
- TimerTask task = new TimerTask() {
- @Override
- public void run() {
- try {
- rotateOutputFile();
- } catch(IOException e){
- LOG.warn("IOException during scheduled file rotation.", e);
- }
- }
- };
- this.rotationTimer.scheduleAtFixedRate(task, interval, interval);
- }
- }
-
- protected void rotateOutputFile() throws IOException {
- LOG.info("Rotating output file...");
- long start = System.currentTimeMillis();
- synchronized (this.writeLock) {
- closeOutputFile();
- this.rotation++;
-
- Path newFile = createOutputFile();
- LOG.info("Performing " + this.rotationActions.size() + " file rotation actions." );
- for (RotationAction action : this.rotationActions) {
- action.execute(this.fs, this.currentFile);
- }
- this.currentFile = newFile;
- }
- long time = System.currentTimeMillis() - start;
- LOG.info("File rotation took " + time + " ms.");
- }
-
- private Path createOutputFile() throws IOException {
- Path path = new Path(this.fileNameFormat.getPath(), this.fileNameFormat.getName(this.rotation, System.currentTimeMillis()));
- if(fs.getScheme().equals("file")) {
- //in the situation where we're running this in a local filesystem, flushing doesn't work.
- fs.mkdirs(path.getParent());
- this.out = new FSDataOutputStream(new FileOutputStream(path.toString()), null);
- }
- else {
- this.out = this.fs.create(path);
- }
- return path;
- }
-
- private void closeOutputFile() throws IOException {
- this.out.close();
- }
-
-
- public void close() {
- try {
- closeOutputFile();
- } catch (IOException e) {
- throw new RuntimeException("Unable to close output file.", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/main/scripts/start_enrichment_topology.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/scripts/start_enrichment_topology.sh b/metron-platform/metron-enrichment/src/main/scripts/start_enrichment_topology.sh
new file mode 100755
index 0000000..6824b87
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/scripts/start_enrichment_topology.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+METRON_VERSION=${project.version}
+METRON_HOME=/usr/metron/$METRON_VERSION
+TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION-uber.jar
+storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/enrichment/remote.yaml --filter $METRON_HOME/config/enrichment.properties
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
index 04415e1..c42c284 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
@@ -21,11 +21,10 @@ import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.EnrichmentConfigurations;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.test.bolt.BaseEnrichmentBoltTest;
-import org.apache.metron.common.configuration.Configurations;
import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.writer.bolt.BulkMessageWriterBolt;
import org.hamcrest.Description;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
new file mode 100644
index 0000000..6b12264
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
@@ -0,0 +1,426 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.integration;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.google.common.base.*;
+
+import com.google.common.collect.Iterables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.metron.TestConstants;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.enrichment.converter.EnrichmentKey;
+import org.apache.metron.enrichment.converter.EnrichmentValue;
+import org.apache.metron.enrichment.converter.EnrichmentHelper;
+import org.apache.metron.integration.*;
+import org.apache.metron.enrichment.integration.components.ConfigUploadComponent;
+import org.apache.metron.integration.utils.TestUtils;
+import org.apache.metron.test.utils.UnitTestHelper;
+import org.apache.metron.integration.components.FluxTopologyComponent;
+import org.apache.metron.integration.components.KafkaWithZKComponent;
+import org.apache.metron.enrichment.integration.mock.MockGeoAdapter;
+import org.apache.metron.test.mock.MockHTable;
+import org.apache.metron.enrichment.lookup.LookupKV;
+
+import org.apache.metron.enrichment.integration.utils.SampleUtil;
+import org.apache.metron.common.utils.JSONUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+public class EnrichmentIntegrationTest extends BaseIntegrationTest {
+
+ private static final String SRC_IP = "ip_src_addr";
+ private static final String DST_IP = "ip_dst_addr";
+ private static final String MALICIOUS_IP_TYPE = "malicious_ip";
+ private static final String PLAYFUL_CLASSIFICATION_TYPE = "playful_classification";
+ private static final Map<String, Object> PLAYFUL_ENRICHMENT = new HashMap<String, Object>() {{
+ put("orientation", "north");
+ }};
+ protected String testSensorType = "test";
+ protected String hdfsDir = "target/enrichmentIntegrationTest/hdfs";
+ protected String fluxPath = "../metron-enrichment/src/main/flux/enrichment/test.yaml";
+ protected String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
+ private String sampleIndexedPath = TestConstants.SAMPLE_DATA_INDEXED_PATH + "TestIndexed";
+
+ public static class Provider implements TableProvider, Serializable {
+ MockHTable.Provider provider = new MockHTable.Provider();
+ @Override
+ public HTableInterface getTable(Configuration config, String tableName) throws IOException {
+ return provider.getTable(config, tableName);
+ }
+ }
+
+
+
+
+
+
+ @Test
+ public void test() throws Exception {
+ final EnrichmentConfigurations configurations = SampleUtil.getSampleEnrichmentConfigs();
+ final String dateFormat = "yyyy.MM.dd.HH";
+ final List<byte[]> inputMessages = TestUtils.readSampleData(sampleParsedPath);
+ final String cf = "cf";
+ final String trackerHBaseTableName = "tracker";
+ final String threatIntelTableName = "threat_intel";
+ final String enrichmentsTableName = "enrichments";
+ final Properties topologyProperties = new Properties() {{
+ setProperty("org.apache.metron.enrichment.host.known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"},\n" +
+ "{\"ip\":\"10.1.128.237\", \"local\":\"UNKNOWN\", \"type\":\"unknown\", \"asset_value\" : \"important\"},\n" +
+ "{\"ip\":\"10.60.10.254\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"},\n" +
+ "{\"ip\":\"10.0.2.15\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}]");
+ setProperty("hbase.provider.impl","" + Provider.class.getName());
+ setProperty("threat.intel.tracker.table", trackerHBaseTableName);
+ setProperty("threat.intel.tracker.cf", cf);
+ setProperty("threat.intel.simple.hbase.table", threatIntelTableName);
+ setProperty("threat.intel.simple.hbase.cf", cf);
+ setProperty("enrichment.simple.hbase.table", enrichmentsTableName);
+ setProperty("enrichment.simple.hbase.cf", cf);
+ setProperty("enrichment.output.topic", Constants.INDEXING_TOPIC);
+ }};
+ final KafkaWithZKComponent kafkaComponent = getKafkaComponent(topologyProperties, new ArrayList<KafkaWithZKComponent.Topic>() {{
+ add(new KafkaWithZKComponent.Topic(Constants.ENRICHMENT_TOPIC, 1));
+ add(new KafkaWithZKComponent.Topic(Constants.INDEXING_TOPIC, 1));
+ }});
+
+ ConfigUploadComponent configUploadComponent = new ConfigUploadComponent()
+ .withTopologyProperties(topologyProperties)
+ .withGlobalConfigsPath(TestConstants.SAMPLE_CONFIG_PATH)
+ .withEnrichmentConfigsPath(TestConstants.SAMPLE_CONFIG_PATH);
+
+ //create MockHBaseTables
+ final MockHTable trackerTable = (MockHTable)MockHTable.Provider.addToCache(trackerHBaseTableName, cf);
+ final MockHTable threatIntelTable = (MockHTable)MockHTable.Provider.addToCache(threatIntelTableName, cf);
+ EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>(){{
+ add(new LookupKV<>(new EnrichmentKey(MALICIOUS_IP_TYPE, "10.0.2.3"), new EnrichmentValue(new HashMap<>())));
+ }});
+ final MockHTable enrichmentTable = (MockHTable)MockHTable.Provider.addToCache(enrichmentsTableName, cf);
+ EnrichmentHelper.INSTANCE.load(enrichmentTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>(){{
+ add(new LookupKV<>(new EnrichmentKey(PLAYFUL_CLASSIFICATION_TYPE, "10.0.2.3")
+ , new EnrichmentValue(PLAYFUL_ENRICHMENT )
+ )
+ );
+ }});
+ FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder()
+ .withTopologyLocation(new File(fluxPath))
+ .withTopologyName("test")
+ .withTopologyProperties(topologyProperties)
+ .build();
+
+
+ UnitTestHelper.verboseLogging();
+ ComponentRunner runner = new ComponentRunner.Builder()
+ .withComponent("kafka", kafkaComponent)
+ .withComponent("config", configUploadComponent)
+ .withComponent("storm", fluxComponent)
+ .withMillisecondsBetweenAttempts(15000)
+ .withNumRetries(10)
+ .build();
+ runner.start();
+
+ try {
+ fluxComponent.submitTopology();
+
+ kafkaComponent.writeMessages(Constants.ENRICHMENT_TOPIC, inputMessages);
+ List<Map<String, Object>> docs = runner.process(getProcessor(inputMessages));
+ Assert.assertEquals(inputMessages.size(), docs.size());
+ List<Map<String, Object>> cleanedDocs = docs;
+ validateAll(cleanedDocs);
+ }
+ finally {
+ runner.stop();
+ }
+ }
+
+
+ public static void validateAll(List<Map<String, Object>> docs) {
+
+ for (Map<String, Object> doc : docs) {
+ baseValidation(doc);
+ hostEnrichmentValidation(doc);
+ geoEnrichmentValidation(doc);
+ threatIntelValidation(doc);
+ simpleEnrichmentValidation(doc);
+ }
+ }
+
+ public static void baseValidation(Map<String, Object> jsonDoc) {
+ assertEnrichmentsExists("threatintels.", setOf("hbaseThreatIntel"), jsonDoc.keySet());
+ assertEnrichmentsExists("enrichments.", setOf("geo", "host", "hbaseEnrichment" ), jsonDoc.keySet());
+ for(Map.Entry<String, Object> kv : jsonDoc.entrySet()) {
+ //ensure no values are empty.
+ Assert.assertTrue(kv.getValue().toString().length() > 0);
+ }
+ //ensure we always have a source ip and destination ip
+ Assert.assertNotNull(jsonDoc.get(SRC_IP));
+ Assert.assertNotNull(jsonDoc.get(DST_IP));
+ }
+
+ private static class EvaluationPayload {
+ Map<String, Object> indexedDoc;
+ String key;
+ public EvaluationPayload(Map<String, Object> indexedDoc, String key) {
+ this.indexedDoc = indexedDoc;
+ this.key = key;
+ }
+ }
+
+ private static enum HostEnrichments implements Predicate<EvaluationPayload>{
+
+ LOCAL_LOCATION(new Predicate<EvaluationPayload>() {
+
+ @Override
+ public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+
+ return evaluationPayload.indexedDoc.getOrDefault("enrichments.host." + evaluationPayload.key + ".known_info.local","").equals("YES");
+
+ }
+ })
+
+ ,UNKNOWN_LOCATION(new Predicate<EvaluationPayload>() {
+
+ @Override
+ public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+ return evaluationPayload.indexedDoc.getOrDefault("enrichments.host." + evaluationPayload.key + ".known_info.local","").equals("UNKNOWN");
+ }
+ })
+ ,IMPORTANT(new Predicate<EvaluationPayload>() {
+ @Override
+ public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+ return evaluationPayload.indexedDoc.getOrDefault("enrichments.host." + evaluationPayload.key + ".known_info.asset_value","").equals("important");
+ }
+ })
+ ,PRINTER_TYPE(new Predicate<EvaluationPayload>() {
+ @Override
+ public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+ return evaluationPayload.indexedDoc.getOrDefault("enrichments.host." + evaluationPayload.key + ".known_info.type","").equals("printer");
+ }
+ })
+ ,WEBSERVER_TYPE(new Predicate<EvaluationPayload>() {
+ @Override
+ public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+ return evaluationPayload.indexedDoc.getOrDefault("enrichments.host." + evaluationPayload.key + ".known_info.type","").equals("webserver");
+ }
+ })
+ ,UNKNOWN_TYPE(new Predicate<EvaluationPayload>() {
+ @Override
+ public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+ return evaluationPayload.indexedDoc.getOrDefault("enrichments.host." + evaluationPayload.key + ".known_info.type","").equals("unknown");
+ }
+ })
+ ;
+
+ Predicate<EvaluationPayload> _predicate;
+ HostEnrichments(Predicate<EvaluationPayload> predicate) {
+ this._predicate = predicate;
+ }
+
+ public boolean apply(EvaluationPayload payload) {
+ return _predicate.apply(payload);
+ }
+
+ }
+
+ private static void assertEnrichmentsExists(String topLevel, Set<String> expectedEnrichments, Set<String> keys) {
+ for(String key : keys) {
+ if(key.startsWith(topLevel)) {
+ String secondLevel = Iterables.get(Splitter.on(".").split(key), 1);
+ String message = "Found an enrichment/threat intel (" + secondLevel + ") that I didn't expect (expected enrichments :"
+ + Joiner.on(",").join(expectedEnrichments) + "), but it was not there. If you've created a new"
+ + " enrichment, then please add a validation method to this unit test. Otherwise, it's a solid error"
+ + " and should be investigated.";
+ Assert.assertTrue( message, expectedEnrichments.contains(secondLevel));
+ }
+ }
+ }
+ private static void simpleEnrichmentValidation(Map<String, Object> indexedDoc) {
+ if(indexedDoc.getOrDefault(SRC_IP,"").equals("10.0.2.3")
+ || indexedDoc.getOrDefault(DST_IP,"").equals("10.0.2.3")
+ ) {
+ Assert.assertTrue(keyPatternExists("enrichments.hbaseEnrichment", indexedDoc));
+ if(indexedDoc.getOrDefault(SRC_IP,"").equals("10.0.2.3")) {
+ Assert.assertEquals(indexedDoc.get("enrichments.hbaseEnrichment." + SRC_IP + "." + PLAYFUL_CLASSIFICATION_TYPE+ ".orientation")
+ , PLAYFUL_ENRICHMENT.get("orientation")
+ );
+ }
+ else if(indexedDoc.getOrDefault(DST_IP,"").equals("10.0.2.3")) {
+ Assert.assertEquals( indexedDoc.get("enrichments.hbaseEnrichment." + DST_IP + "." + PLAYFUL_CLASSIFICATION_TYPE + ".orientation")
+ , PLAYFUL_ENRICHMENT.get("orientation")
+ );
+ }
+ }
+
+ }
+ private static void threatIntelValidation(Map<String, Object> indexedDoc) {
+ if(indexedDoc.getOrDefault(SRC_IP,"").equals("10.0.2.3")
+ || indexedDoc.getOrDefault(DST_IP,"").equals("10.0.2.3")
+ ) {
+ //if we have any threat intel messages, we want to tag is_alert to true
+ Assert.assertTrue(keyPatternExists("threatintels.", indexedDoc));
+ Assert.assertTrue(indexedDoc.containsKey("threat.triage.level"));
+ Assert.assertEquals(indexedDoc.getOrDefault("is_alert",""), "true");
+ Assert.assertEquals((double)indexedDoc.get("threat.triage.level"), 10d, 1e-7);
+ }
+ else {
+ //For YAF this is the case, but if we do snort later on, this will be invalid.
+ Assert.assertNull(indexedDoc.get("is_alert"));
+ Assert.assertFalse(keyPatternExists("threatintels.", indexedDoc));
+ }
+ //ip threat intels
+ if(keyPatternExists("threatintels.hbaseThreatIntel.", indexedDoc)) {
+ if(indexedDoc.getOrDefault(SRC_IP,"").equals("10.0.2.3")) {
+ Assert.assertEquals(indexedDoc.get("threatintels.hbaseThreatIntel." + SRC_IP + "." + MALICIOUS_IP_TYPE), "alert");
+ }
+ else if(indexedDoc.getOrDefault(DST_IP,"").equals("10.0.2.3")) {
+ Assert.assertEquals(indexedDoc.get("threatintels.hbaseThreatIntel." + DST_IP + "." + MALICIOUS_IP_TYPE), "alert");
+ }
+ else {
+ Assert.fail("There was a threat intels that I did not expect: " + indexedDoc);
+ }
+ }
+
+ }
+
+ private static void geoEnrichmentValidation(Map<String, Object> indexedDoc) {
+ //should have geo enrichment on every message due to mock geo adapter
+ Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".location_point"), MockGeoAdapter.DEFAULT_LOCATION_POINT);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP +".location_point"), MockGeoAdapter.DEFAULT_LOCATION_POINT);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".longitude"), MockGeoAdapter.DEFAULT_LONGITUDE);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".longitude"), MockGeoAdapter.DEFAULT_LONGITUDE);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".city"), MockGeoAdapter.DEFAULT_CITY);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".city"), MockGeoAdapter.DEFAULT_CITY);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".latitude"), MockGeoAdapter.DEFAULT_LATITUDE);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".latitude"), MockGeoAdapter.DEFAULT_LATITUDE);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".country"), MockGeoAdapter.DEFAULT_COUNTRY);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".country"), MockGeoAdapter.DEFAULT_COUNTRY);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".dmaCode"), MockGeoAdapter.DEFAULT_DMACODE);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".dmaCode"), MockGeoAdapter.DEFAULT_DMACODE);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".postalCode"), MockGeoAdapter.DEFAULT_POSTAL_CODE);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".postalCode"), MockGeoAdapter.DEFAULT_POSTAL_CODE);
+ }
+
+ private static void hostEnrichmentValidation(Map<String, Object> indexedDoc) {
+ boolean enriched = false;
+ //important local printers
+ {
+ Set<String> ips = setOf("10.0.2.15", "10.60.10.254");
+ if (ips.contains(indexedDoc.get(SRC_IP))) {
+ //this is a local, important, printer
+ Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
+ ,HostEnrichments.IMPORTANT
+ ,HostEnrichments.PRINTER_TYPE
+ ).apply(new EvaluationPayload(indexedDoc, SRC_IP))
+ );
+ enriched = true;
+ }
+ if (ips.contains(indexedDoc.get(DST_IP))) {
+ Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
+ ,HostEnrichments.IMPORTANT
+ ,HostEnrichments.PRINTER_TYPE
+ ).apply(new EvaluationPayload(indexedDoc, DST_IP))
+ );
+ enriched = true;
+ }
+ }
+ //important local webservers
+ {
+ Set<String> ips = setOf("10.1.128.236");
+ if (ips.contains(indexedDoc.get(SRC_IP))) {
+ //this is a local, important, printer
+ Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
+ ,HostEnrichments.IMPORTANT
+ ,HostEnrichments.WEBSERVER_TYPE
+ ).apply(new EvaluationPayload(indexedDoc, SRC_IP))
+ );
+ enriched = true;
+ }
+ if (ips.contains(indexedDoc.get(DST_IP))) {
+ Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
+ ,HostEnrichments.IMPORTANT
+ ,HostEnrichments.WEBSERVER_TYPE
+ ).apply(new EvaluationPayload(indexedDoc, DST_IP))
+ );
+ enriched = true;
+ }
+ }
+ if(!enriched) {
+ Assert.assertFalse(keyPatternExists("enrichments.host", indexedDoc));
+ }
+ }
+
+
+ private static boolean keyPatternExists(String pattern, Map<String, Object> indexedObj) {
+ for(String k : indexedObj.keySet()) {
+ if(k.startsWith(pattern)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ private static Set<String> setOf(String... items) {
+ Set<String> ret = new HashSet<>();
+ for(String item : items) {
+ ret.add(item);
+ }
+ return ret;
+ }
+
+ public Processor<List<Map<String, Object>>> getProcessor(List<byte[]> inputMessages) {
+ return new Processor<List<Map<String, Object>>>() {
+ List<Map<String, Object>> docs = null;
+
+ public ReadinessState process(ComponentRunner runner) {
+ KafkaWithZKComponent kafkaComponent = runner.getComponent("kafka", KafkaWithZKComponent.class);
+ List<byte[]> messages = kafkaComponent.readMessages(Constants.INDEXING_TOPIC);
+ if (messages.size() == inputMessages.size()) {
+ docs = new ArrayList<>();
+ for(byte[] message : messages) {
+ try {
+ docs.add(JSONUtils.INSTANCE.load(new String(message), new TypeReference<Map<String, Object>>() {}));
+ } catch (IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+ return ReadinessState.READY;
+ } else {
+ return ReadinessState.NOT_READY;
+ }
+ }
+
+ public List<Map<String, Object>> getResult() {
+ return docs;
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java
new file mode 100644
index 0000000..5cc14ac
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/components/ConfigUploadComponent.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.integration.components;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.integration.InMemoryComponent;
+import org.apache.metron.integration.UnableToStartException;
+import org.apache.metron.integration.components.KafkaWithZKComponent;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+
+public class ConfigUploadComponent implements InMemoryComponent {
+
+ private Properties topologyProperties;
+ private String globalConfigPath;
+ private String parserConfigsPath;
+ private String enrichmentConfigsPath;
+ private Optional<String> globalConfig = Optional.empty();
+ private Map<String, SensorParserConfig> parserSensorConfigs = new HashMap<>();
+ public ConfigUploadComponent withTopologyProperties(Properties topologyProperties) {
+ this.topologyProperties = topologyProperties;
+ return this;
+ }
+
+ public ConfigUploadComponent withGlobalConfigsPath(String globalConfigPath) {
+ this.globalConfigPath = globalConfigPath;
+ return this;
+ }
+
+ public ConfigUploadComponent withParserConfigsPath(String parserConfigsPath) {
+ this.parserConfigsPath = parserConfigsPath;
+ return this;
+ }
+ public ConfigUploadComponent withEnrichmentConfigsPath(String enrichmentConfigsPath) {
+ this.enrichmentConfigsPath = enrichmentConfigsPath;
+ return this;
+ }
+
+ public ConfigUploadComponent withParserSensorConfig(String name, SensorParserConfig config) {
+ parserSensorConfigs.put(name, config);
+ return this;
+ }
+
+ public ConfigUploadComponent withGlobalConfig(String globalConfig) {
+ this.globalConfig = Optional.ofNullable(globalConfig);
+ return this;
+ }
+
+ @Override
+ public void start() throws UnableToStartException {
+ try {
+ if(globalConfigPath != null) {
+ ConfigurationsUtils.uploadConfigsToZookeeper(globalConfigPath
+ , parserConfigsPath
+ , enrichmentConfigsPath
+ , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
+ );
+ }
+ for(Map.Entry<String, SensorParserConfig> kv : parserSensorConfigs.entrySet()) {
+ ConfigurationsUtils.writeSensorParserConfigToZookeeper( kv.getKey()
+ , kv.getValue()
+ , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
+ );
+ }
+ if(globalConfig.isPresent()) {
+ ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig.get().getBytes()
+ , topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY)
+ );
+ }
+
+ } catch (Exception e) {
+ throw new UnableToStartException(e.getMessage(), e);
+ }
+ }
+
+ public SensorParserConfig getSensorParserConfig(String sensorType) {
+ SensorParserConfig sensorParserConfig = new SensorParserConfig();
+ CuratorFramework client = ConfigurationsUtils.getClient(topologyProperties.getProperty(KafkaWithZKComponent.ZOOKEEPER_PROPERTY));
+ client.start();
+ try {
+ sensorParserConfig = ConfigurationsUtils.readSensorParserConfigFromZookeeper(sensorType, client);
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ client.close();
+ }
+ return sensorParserConfig;
+ }
+
+ @Override
+ public void stop() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockGeoAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockGeoAdapter.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockGeoAdapter.java
new file mode 100644
index 0000000..14a4493
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockGeoAdapter.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.integration.mock;
+
+import com.google.common.base.Joiner;
+import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
+import org.json.simple.JSONObject;
+
+import java.io.Serializable;
+
+public class MockGeoAdapter implements EnrichmentAdapter<CacheKey>,
+ Serializable {
+
+ public static final String DEFAULT_LOC_ID = "1";
+ public static final String DEFAULT_COUNTRY = "test country";
+ public static final String DEFAULT_CITY = "test city";
+ public static final String DEFAULT_POSTAL_CODE = "test postalCode";
+ public static final String DEFAULT_LATITUDE = "test latitude";
+ public static final String DEFAULT_LONGITUDE = "test longitude";
+ public static final String DEFAULT_DMACODE= "test dmaCode";
+ public static final String DEFAULT_LOCATION_POINT= Joiner.on(',').join(DEFAULT_LATITUDE,DEFAULT_LONGITUDE);
+
+ @Override
+ public void logAccess(CacheKey value) {
+
+ }
+
+ public JSONObject enrich(CacheKey cache ) {
+ JSONObject enriched = new JSONObject();
+ enriched.put("locID", DEFAULT_LOC_ID);
+ enriched.put("country", DEFAULT_COUNTRY);
+ enriched.put("city", DEFAULT_CITY);
+ enriched.put("postalCode", DEFAULT_POSTAL_CODE);
+ enriched.put("latitude", DEFAULT_LATITUDE);
+ enriched.put("longitude", DEFAULT_LONGITUDE);
+ enriched.put("dmaCode", DEFAULT_DMACODE);
+ enriched.put("location_point", DEFAULT_LOCATION_POINT);
+ return enriched;
+ }
+
+ public boolean initializeAdapter() {
+ return true;
+ }
+
+ public void cleanup() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockHBaseConnector.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockHBaseConnector.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockHBaseConnector.java
new file mode 100644
index 0000000..3b16b6a
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockHBaseConnector.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.integration.mock;
+
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
+import org.apache.metron.hbase.Connector;
+import org.apache.metron.hbase.TupleTableConfig;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class MockHBaseConnector extends Connector {
+ static List<Put> puts = Collections.synchronizedList(new ArrayList<Put>());
+ public MockHBaseConnector(TupleTableConfig conf, String _quorum, String _port) throws IOException {
+ super(conf, _quorum, _port);
+ }
+
+ @Override
+ public void put(Put put) throws InterruptedIOException, RetriesExhaustedWithDetailsException {
+ puts.add(put);
+ }
+
+ @Override
+ public void close() {
+
+ }
+ public static void clear() {
+ puts.clear();
+ }
+ public static List<Put> getPuts() {
+ return puts;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockTableProvider.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockTableProvider.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockTableProvider.java
new file mode 100644
index 0000000..ac2904a
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/mock/MockTableProvider.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.enrichment.integration.mock;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.test.mock.MockHTable;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+public class MockTableProvider implements TableProvider, Serializable {
+ static MockHTable.Provider provider = new MockHTable.Provider();
+ @Override
+ public HTableInterface getTable(Configuration config, String tableName) throws IOException {
+ return provider.getTable(config, tableName);
+ }
+ public static void addTable(String tableName, String... cf) {
+ provider.addToCache(tableName, cf);
+ }
+ public static MockHTable getTable(String tableName) {
+ try {
+ return (MockHTable) provider.getTable(null, tableName);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to get table: " + tableName);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5ffcef8d/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java
new file mode 100644
index 0000000..40dcfe4
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/utils/SampleUtil.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.integration.utils;
+
+import org.apache.metron.TestConstants;
+import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.common.configuration.ParserConfigurations;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class SampleUtil {
+
+ public static Configurations getSampleConfigs() throws IOException {
+ Configurations configurations = new Configurations();
+ configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
+ return configurations;
+ }
+
+ public static ParserConfigurations getSampleParserConfigs() throws IOException {
+ ParserConfigurations configurations = new ParserConfigurations();
+ configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
+ Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH);
+ for(String sensorType: sensorParserConfigs.keySet()) {
+ configurations.updateSensorParserConfig(sensorType, sensorParserConfigs.get(sensorType));
+ }
+ return configurations;
+ }
+
+ public static EnrichmentConfigurations getSampleEnrichmentConfigs() throws IOException {
+ EnrichmentConfigurations configurations = new EnrichmentConfigurations();
+ configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
+ Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.SAMPLE_CONFIG_PATH);
+ for(String sensorType: sensorEnrichmentConfigs.keySet()) {
+ configurations.updateSensorEnrichmentConfig(sensorType, sensorEnrichmentConfigs.get(sensorType));
+ }
+ return configurations;
+ }
+
+}