You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2019/04/19 19:11:29 UTC
[metron] branch master updated: METRON-2078 Remove Storm dependency
from metron-writer (merrimanr) closes apache/metron#1386
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 51d1c81 METRON-2078 Remove Storm dependency from metron-writer (merrimanr) closes apache/metron#1386
51d1c81 is described below
commit 51d1c812c1e45f57da8c27fe37fd13797707884e
Author: merrimanr <me...@gmail.com>
AuthorDate: Fri Apr 19 14:11:12 2019 -0500
METRON-2078 Remove Storm dependency from metron-writer (merrimanr) closes apache/metron#1386
---
metron-analytics/metron-profiler-storm/pom.xml | 2 +-
.../metron/storm/common/utils/ErrorUtilsTest.java | 42 ---
.../org/apache/metron/common/utils/ErrorUtils.java | 113 -------
.../apache/metron/common/utils/RuntimeErrors.java | 79 +++++
.../metron/common/utils/RuntimeErrorsTest.java} | 35 +-
.../dataloads/nonbulk/taxii/TaxiiHandler.java | 4 +-
metron-platform/metron-elasticsearch/pom.xml | 2 +-
.../metron-enrichment-storm/pom.xml | 2 +-
metron-platform/metron-indexing/pom.xml | 41 +--
.../integration/HDFSIndexingIntegrationTest.java | 3 +-
metron-platform/metron-management/pom.xml | 4 +-
.../metron-parsing/metron-parsers-common/pom.xml | 2 +-
.../metron-parsing/metron-parsing-storm/pom.xml | 6 +
.../{ => metron-writer-common}/pom.xml | 67 +---
.../org/apache/metron/writer/AbstractWriter.java | 0
.../org/apache/metron/writer/BatchSizePolicy.java | 0
.../apache/metron/writer/BatchTimeoutPolicy.java | 0
.../apache/metron/writer/BulkWriterComponent.java | 0
.../java/org/apache/metron/writer/FlushPolicy.java | 0
.../java/org/apache/metron/writer/NoopWriter.java | 0
.../apache/metron/writer/WriterToBulkWriter.java | 0
.../writer/hbase/SimpleHbaseEnrichmentWriter.java | 0
.../apache/metron/writer/kafka/KafkaWriter.java | 0
.../src/main/resources/META-INF/LICENSE | 0
.../src/main/resources/META-INF/NOTICE | 0
.../apache/metron/writer/BatchSizePolicyTest.java | 0
.../metron/writer/BatchTimeoutPolicyTest.java | 0
.../metron/writer/BulkWriterComponentTest.java | 4 +-
.../org/apache/metron/writer/NoopWriterTest.java | 0
.../metron/writer/kafka/KafkaWriterTest.java | 0
.../src/test/resources/log4j.properties | 0
.../metron-writer/metron-writer-storm/pom.xml | 93 ++++++
.../org/apache/metron/writer/AckTuplesPolicy.java | 0
.../metron/writer/bolt/BatchTimeoutHelper.java | 0
.../metron/writer/bolt/BulkMessageWriterBolt.java | 0
.../writer/hdfs/ClonedSyncPolicyCreator.java | 0
.../org/apache/metron/writer/hdfs/HdfsWriter.java | 0
.../writer/hdfs/PathExtensionFileNameFormat.java | 0
.../metron/writer/hdfs/SourceAwareMoveAction.java | 0
.../apache/metron/writer/hdfs/SourceHandler.java | 0
.../metron/writer/hdfs/SourceHandlerCallback.java | 0
.../metron/writer/hdfs/SourceHandlerKey.java | 0
.../metron/writer/hdfs/SyncPolicyCreator.java | 0
.../apache/metron/writer/AckTuplesPolicyTest.java | 17 +-
.../metron/writer/bolt/BatchTimeoutHelperTest.java | 0
.../writer/bolt/BulkMessageWriterBoltTest.java | 52 +--
.../writer/hdfs/ClonedSyncPolicyCreatorTest.java | 0
.../apache/metron/writer/hdfs/HdfsWriterTest.java | 0
.../hdfs/PathExtensionFileNameFormatTest.java | 0
.../metron/writer/hdfs/SourceHandlerTest.java | 0
metron-platform/metron-writer/pom.xml | 366 ++-------------------
51 files changed, 270 insertions(+), 664 deletions(-)
diff --git a/metron-analytics/metron-profiler-storm/pom.xml b/metron-analytics/metron-profiler-storm/pom.xml
index 66deb4d..fe615e2 100644
--- a/metron-analytics/metron-profiler-storm/pom.xml
+++ b/metron-analytics/metron-profiler-storm/pom.xml
@@ -84,7 +84,7 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-writer</artifactId>
+ <artifactId>metron-writer-storm</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
diff --git a/metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/utils/ErrorUtilsTest.java b/metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/utils/ErrorUtilsTest.java
index 037a06c..7d0694e 100644
--- a/metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/utils/ErrorUtilsTest.java
+++ b/metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/utils/ErrorUtilsTest.java
@@ -19,17 +19,10 @@ package org.apache.metron.storm.common.utils;
import org.apache.metron.common.Constants;
import org.apache.metron.common.error.MetronError;
-import org.apache.metron.common.utils.ErrorUtils;
import org.apache.metron.test.error.MetronErrorJSONMatcher;
import org.apache.storm.task.OutputCollector;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import java.io.IOException;
-
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.nullValue;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.argThat;
import static org.mockito.Matchers.eq;
@@ -39,41 +32,6 @@ import static org.mockito.Mockito.verify;
public class ErrorUtilsTest {
- @Rule
- public ExpectedException exception = ExpectedException.none();
-
- @Test
- public void illegal_arg_throws_exception_with_reason() throws Exception {
- exception.expect(IllegalArgumentException.class);
- exception.expectMessage("illegal arg happened");
- exception.expectCause(nullValue(Throwable.class));
- ErrorUtils.RuntimeErrors.ILLEGAL_ARG.throwRuntime("illegal arg happened");
- }
-
- @Test
- public void illegal_arg_throws_exception_with_reason_and_cause() throws Exception {
- exception.expect(IllegalArgumentException.class);
- exception.expectMessage("illegal arg happened");
- exception.expectCause(instanceOf(IOException.class));
- ErrorUtils.RuntimeErrors.ILLEGAL_ARG.throwRuntime("illegal arg happened", new IOException("bad io"));
- }
-
- @Test
- public void illegal_state_throws_exception_with_reason() throws Exception {
- exception.expect(IllegalStateException.class);
- exception.expectMessage("illegal state happened");
- exception.expectCause(nullValue(Throwable.class));
- ErrorUtils.RuntimeErrors.ILLEGAL_STATE.throwRuntime("illegal state happened");
- }
-
- @Test
- public void illegal_state_throws_exception_with_reason_and_cause() throws Exception {
- exception.expect(IllegalStateException.class);
- exception.expectMessage("illegal state happened");
- exception.expectCause(instanceOf(IOException.class));
- ErrorUtils.RuntimeErrors.ILLEGAL_STATE.throwRuntime("illegal state happened", new IOException("bad io"));
- }
-
@Test
public void handleErrorShouldEmitAndReportError() throws Exception {
Throwable e = new Exception("error");
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java
deleted file mode 100644
index bd666c3..0000000
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.common.utils;
-
-import static java.lang.String.format;
-
-import java.lang.invoke.MethodHandles;
-import java.lang.management.ManagementFactory;
-import java.lang.management.ThreadInfo;
-import java.lang.management.ThreadMXBean;
-import java.util.Optional;
-import java.util.function.Function;
-import org.apache.commons.lang3.tuple.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ErrorUtils {
- private final static Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public enum RuntimeErrors {
- ILLEGAL_ARG(t -> new IllegalArgumentException(formatReason(t), t.getRight().orElse(null))),
- ILLEGAL_STATE(t -> new IllegalStateException(formatReason(t), t.getRight().orElse(null)));
-
- Function<Pair<String, Optional<Throwable>>, RuntimeException> func;
-
- RuntimeErrors(Function<Pair<String, Optional<Throwable>>, RuntimeException> func) {
- this.func = func;
- }
-
- /**
- * Throw runtime exception with "reason".
- *
- * @param reason Message to include in exception
- */
- public void throwRuntime(String reason) {
- throwRuntime(reason, Optional.empty());
- }
-
- /**
- * Throw runtime exception with format "reason + cause message + cause Throwable".
- *
- * @param reason Message to include in exception
- * @param t Wrapped exception
- */
- public void throwRuntime(String reason, Throwable t) {
- throwRuntime(reason, Optional.of(t));
- }
-
- /**
- * Throw runtime exception with format "reason + cause message + cause Throwable".
- * If the optional Throwable is empty/null, the exception will only include "reason".
- *
- * @param reason Message to include in exception
- * @param t Optional wrapped exception
- */
- public void throwRuntime(String reason, Optional<Throwable> t) {
- throw func.apply(Pair.of(reason, t));
- }
-
- private static String formatReason(Pair<String, Optional<Throwable>> p) {
- return formatReason(p.getLeft(), p.getRight());
- }
-
- private static String formatReason(String reason, Optional<Throwable> t) {
- if (t.isPresent()) {
- return format("%s - reason:%s", reason, t.get());
- } else {
- return format("%s", reason);
- }
- }
- }
-
- /**
- * Generates a string version of a thread dump.
- *
- * @return String of the thread dump
- */
- public static String generateThreadDump() {
- final StringBuilder dump = new StringBuilder();
- final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
- final ThreadInfo[] threadInfos = threadMXBean.getThreadInfo(threadMXBean.getAllThreadIds(), 100);
- for (ThreadInfo threadInfo : threadInfos) {
- dump.append('"');
- dump.append(threadInfo.getThreadName());
- dump.append("\" ");
- final Thread.State state = threadInfo.getThreadState();
- dump.append("\n java.lang.Thread.State: ");
- dump.append(state);
- final StackTraceElement[] stackTraceElements = threadInfo.getStackTrace();
- for (final StackTraceElement stackTraceElement : stackTraceElements) {
- dump.append("\n at ");
- dump.append(stackTraceElement);
- }
- dump.append("\n\n");
- }
- return dump.toString();
- }
-}
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/RuntimeErrors.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/RuntimeErrors.java
new file mode 100644
index 0000000..266c67c
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/RuntimeErrors.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.utils;
+
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.util.Optional;
+import java.util.function.Function;
+
+import static java.lang.String.format;
+
+public enum RuntimeErrors {
+ ILLEGAL_ARG(t -> new IllegalArgumentException(formatReason(t), t.getRight().orElse(null))),
+ ILLEGAL_STATE(t -> new IllegalStateException(formatReason(t), t.getRight().orElse(null)));
+
+ Function<Pair<String, Optional<Throwable>>, RuntimeException> func;
+
+ RuntimeErrors(Function<Pair<String, Optional<Throwable>>, RuntimeException> func) {
+ this.func = func;
+ }
+
+ /**
+ * Throw runtime exception with "reason".
+ *
+ * @param reason Message to include in exception
+ */
+ public void throwRuntime(String reason) {
+ throwRuntime(reason, Optional.empty());
+ }
+
+ /**
+ * Throw runtime exception with format "reason + cause message + cause Throwable".
+ *
+ * @param reason Message to include in exception
+ * @param t Wrapped exception
+ */
+ public void throwRuntime(String reason, Throwable t) {
+ throwRuntime(reason, Optional.of(t));
+ }
+
+ /**
+ * Throw runtime exception with format "reason + cause message + cause Throwable".
+ * If the optional Throwable is empty/null, the exception will only include "reason".
+ *
+ * @param reason Message to include in exception
+ * @param t Optional wrapped exception
+ */
+ public void throwRuntime(String reason, Optional<Throwable> t) {
+ throw func.apply(Pair.of(reason, t));
+ }
+
+ private static String formatReason(Pair<String, Optional<Throwable>> p) {
+ return formatReason(p.getLeft(), p.getRight());
+ }
+
+ private static String formatReason(String reason, Optional<Throwable> t) {
+ if (t.isPresent()) {
+ return format("%s - reason:%s", reason, t.get());
+ } else {
+ return format("%s", reason);
+ }
+ }
+}
diff --git a/metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/utils/ErrorUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/RuntimeErrorsTest.java
similarity index 59%
copy from metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/utils/ErrorUtilsTest.java
copy to metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/RuntimeErrorsTest.java
index 037a06c..9bc8929 100644
--- a/metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/utils/ErrorUtilsTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/RuntimeErrorsTest.java
@@ -15,13 +15,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.metron.storm.common.utils;
-import org.apache.metron.common.Constants;
-import org.apache.metron.common.error.MetronError;
-import org.apache.metron.common.utils.ErrorUtils;
-import org.apache.metron.test.error.MetronErrorJSONMatcher;
-import org.apache.storm.task.OutputCollector;
+package org.apache.metron.common.utils;
+
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -30,14 +26,8 @@ import java.io.IOException;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.nullValue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-public class ErrorUtilsTest {
+public class RuntimeErrorsTest {
@Rule
public ExpectedException exception = ExpectedException.none();
@@ -47,7 +37,7 @@ public class ErrorUtilsTest {
exception.expect(IllegalArgumentException.class);
exception.expectMessage("illegal arg happened");
exception.expectCause(nullValue(Throwable.class));
- ErrorUtils.RuntimeErrors.ILLEGAL_ARG.throwRuntime("illegal arg happened");
+ RuntimeErrors.ILLEGAL_ARG.throwRuntime("illegal arg happened");
}
@Test
@@ -55,7 +45,7 @@ public class ErrorUtilsTest {
exception.expect(IllegalArgumentException.class);
exception.expectMessage("illegal arg happened");
exception.expectCause(instanceOf(IOException.class));
- ErrorUtils.RuntimeErrors.ILLEGAL_ARG.throwRuntime("illegal arg happened", new IOException("bad io"));
+ RuntimeErrors.ILLEGAL_ARG.throwRuntime("illegal arg happened", new IOException("bad io"));
}
@Test
@@ -63,7 +53,7 @@ public class ErrorUtilsTest {
exception.expect(IllegalStateException.class);
exception.expectMessage("illegal state happened");
exception.expectCause(nullValue(Throwable.class));
- ErrorUtils.RuntimeErrors.ILLEGAL_STATE.throwRuntime("illegal state happened");
+ RuntimeErrors.ILLEGAL_STATE.throwRuntime("illegal state happened");
}
@Test
@@ -71,17 +61,6 @@ public class ErrorUtilsTest {
exception.expect(IllegalStateException.class);
exception.expectMessage("illegal state happened");
exception.expectCause(instanceOf(IOException.class));
- ErrorUtils.RuntimeErrors.ILLEGAL_STATE.throwRuntime("illegal state happened", new IOException("bad io"));
- }
-
- @Test
- public void handleErrorShouldEmitAndReportError() throws Exception {
- Throwable e = new Exception("error");
- MetronError error = new MetronError().withMessage("error message").withThrowable(e);
- OutputCollector collector = mock(OutputCollector.class);
-
- StormErrorUtils.handleError(collector, error);
- verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
- verify(collector, times(1)).reportError(any());
+ RuntimeErrors.ILLEGAL_STATE.throwRuntime("illegal state happened", new IOException("bad io"));
}
}
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiHandler.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiHandler.java
index 8bbedf2..de795c5 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiHandler.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiHandler.java
@@ -66,7 +66,7 @@ import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.metron.common.utils.ErrorUtils;
+import org.apache.metron.common.utils.RuntimeErrors;
import org.apache.metron.dataloads.extractor.Extractor;
import org.apache.metron.enrichment.converter.EnrichmentConverter;
import org.apache.metron.enrichment.converter.EnrichmentKey;
@@ -188,7 +188,7 @@ public class TaxiiHandler extends TimerTask {
try {
gTime = DatatypeFactory.newInstance().newXMLGregorianCalendar((GregorianCalendar) gc).normalize();
} catch (DatatypeConfigurationException e) {
- ErrorUtils.RuntimeErrors.ILLEGAL_STATE.throwRuntime("Unable to set the begin time due to", e);
+ RuntimeErrors.ILLEGAL_STATE.throwRuntime("Unable to set the begin time due to", e);
}
gTime.setFractionalSecond(null);
LOG.info("Begin Time: {}", gTime);
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index 31c160c..5ff39c4 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -46,7 +46,7 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-writer</artifactId>
+ <artifactId>metron-writer-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
diff --git a/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml b/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml
index 08ae7ac..4d0df7d 100644
--- a/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml
+++ b/metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml
@@ -48,7 +48,7 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-writer</artifactId>
+ <artifactId>metron-writer-storm</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml
index ccb1a58..222e0d2 100644
--- a/metron-platform/metron-indexing/pom.xml
+++ b/metron-platform/metron-indexing/pom.xml
@@ -41,7 +41,7 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-writer</artifactId>
+ <artifactId>metron-writer-storm</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
@@ -101,45 +101,6 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- <version>${global_storm_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-over-slf4j</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-slf4j-impl</artifactId>
- <groupId>org.apache.logging.log4j</groupId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-hdfs</artifactId>
- <version>${global_storm_version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava_version}</version>
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
index 2416c86..2aff2ef 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
@@ -24,7 +24,6 @@ import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.integration.*;
import org.apache.metron.integration.components.KafkaComponent;
import org.apache.metron.integration.utils.TestUtils;
-import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy;
import java.io.File;
import java.io.IOException;
@@ -138,7 +137,7 @@ public class HDFSIndexingIntegrationTest extends IndexingIntegrationTest {
topologyProperties.setProperty("batch_indexing_acker_executors", "0");
topologyProperties.setProperty("batch_indexing_topology_max_spout_pending", "");
topologyProperties.setProperty("batch_indexing_kafka_spout_parallelism", "1");
- topologyProperties.setProperty("bolt_hdfs_rotation_policy", TimedRotationPolicy.class.getCanonicalName());
+ topologyProperties.setProperty("bolt_hdfs_rotation_policy", "org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy");
topologyProperties.setProperty("bolt_hdfs_rotation_policy_count", "1");
topologyProperties.setProperty("bolt_hdfs_rotation_policy_units", "DAYS");
topologyProperties.setProperty("metron_apps_indexed_hdfs_dir", hdfsDir);
diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml
index 4b272ff..a60f920 100644
--- a/metron-platform/metron-management/pom.xml
+++ b/metron-platform/metron-management/pom.xml
@@ -45,7 +45,7 @@
<exclusions>
<exclusion>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-writer</artifactId>
+ <artifactId>metron-writer-common</artifactId>
</exclusion>
</exclusions>
<scope>provided</scope>
@@ -57,7 +57,7 @@
<exclusions>
<exclusion>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-writer</artifactId>
+ <artifactId>metron-writer-common</artifactId>
</exclusion>
</exclusions>
<scope>provided</scope>
diff --git a/metron-platform/metron-parsing/metron-parsers-common/pom.xml b/metron-platform/metron-parsing/metron-parsers-common/pom.xml
index 46c46ca..87df1ab 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/pom.xml
+++ b/metron-platform/metron-parsing/metron-parsers-common/pom.xml
@@ -36,7 +36,7 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-writer</artifactId>
+ <artifactId>metron-writer-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
diff --git a/metron-platform/metron-parsing/metron-parsing-storm/pom.xml b/metron-platform/metron-parsing/metron-parsing-storm/pom.xml
index c936ef9..462c4ac 100644
--- a/metron-platform/metron-parsing/metron-parsing-storm/pom.xml
+++ b/metron-platform/metron-parsing/metron-parsing-storm/pom.xml
@@ -51,6 +51,11 @@
<artifactId>metron-enrichment-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-writer-storm</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
<!-- Storm dependencies -->
<dependency>
@@ -143,6 +148,7 @@
<version>${global_log4j_core_version}</version>
<scope>test</scope>
</dependency>
+
</dependencies>
<build>
<plugins>
diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/metron-writer-common/pom.xml
similarity index 83%
copy from metron-platform/metron-writer/pom.xml
copy to metron-platform/metron-writer/metron-writer-common/pom.xml
index f8ae4c4..ef4fe0e 100644
--- a/metron-platform/metron-writer/pom.xml
+++ b/metron-platform/metron-writer/metron-writer-common/pom.xml
@@ -17,11 +17,11 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-platform</artifactId>
+ <artifactId>metron-writer</artifactId>
<version>0.7.1</version>
</parent>
- <artifactId>metron-writer</artifactId>
- <name>metron-writer</name>
+ <artifactId>metron-writer-common</artifactId>
+ <name>metron-writer-common</name>
<description>Components common to all enrichments</description>
<url>https://metron.apache.org/</url>
<properties>
@@ -134,39 +134,6 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- <version>${global_storm_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-over-slf4j</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-slf4j-impl</artifactId>
- <groupId>org.apache.logging.log4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>${global_httpclient_version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
<version>${global_kafka_version}</version>
@@ -191,21 +158,6 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-hdfs</artifactId>
- <version>${global_storm_version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${global_hadoop_version}</version>
@@ -217,28 +169,21 @@
</exclusions>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-common-storm</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment-common</artifactId>
+ <artifactId>metron-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-test-utilities</artifactId>
+ <artifactId>metron-enrichment-common</artifactId>
<version>${project.parent.version}</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
+ <artifactId>metron-test-utilities</artifactId>
<version>${project.parent.version}</version>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/AbstractWriter.java b/metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/AbstractWriter.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/AbstractWriter.java
rename to metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/AbstractWriter.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BatchSizePolicy.java b/metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/BatchSizePolicy.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BatchSizePolicy.java
rename to metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/BatchSizePolicy.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BatchTimeoutPolicy.java b/metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/BatchTimeoutPolicy.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BatchTimeoutPolicy.java
rename to metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/BatchTimeoutPolicy.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java b/metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
rename to metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/FlushPolicy.java b/metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/FlushPolicy.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/FlushPolicy.java
rename to metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/FlushPolicy.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java b/metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/NoopWriter.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/NoopWriter.java
rename to metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/NoopWriter.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java b/metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java
rename to metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/WriterToBulkWriter.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java b/metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
rename to metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/hbase/SimpleHbaseEnrichmentWriter.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java b/metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java
rename to metron-platform/metron-writer/metron-writer-common/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java
diff --git a/metron-platform/metron-writer/src/main/resources/META-INF/LICENSE b/metron-platform/metron-writer/metron-writer-common/src/main/resources/META-INF/LICENSE
similarity index 100%
rename from metron-platform/metron-writer/src/main/resources/META-INF/LICENSE
rename to metron-platform/metron-writer/metron-writer-common/src/main/resources/META-INF/LICENSE
diff --git a/metron-platform/metron-writer/src/main/resources/META-INF/NOTICE b/metron-platform/metron-writer/metron-writer-common/src/main/resources/META-INF/NOTICE
similarity index 100%
rename from metron-platform/metron-writer/src/main/resources/META-INF/NOTICE
rename to metron-platform/metron-writer/metron-writer-common/src/main/resources/META-INF/NOTICE
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BatchSizePolicyTest.java b/metron-platform/metron-writer/metron-writer-common/src/test/java/org/apache/metron/writer/BatchSizePolicyTest.java
similarity index 100%
rename from metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BatchSizePolicyTest.java
rename to metron-platform/metron-writer/metron-writer-common/src/test/java/org/apache/metron/writer/BatchSizePolicyTest.java
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BatchTimeoutPolicyTest.java b/metron-platform/metron-writer/metron-writer-common/src/test/java/org/apache/metron/writer/BatchTimeoutPolicyTest.java
similarity index 100%
rename from metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BatchTimeoutPolicyTest.java
rename to metron-platform/metron-writer/metron-writer-common/src/test/java/org/apache/metron/writer/BatchTimeoutPolicyTest.java
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java b/metron-platform/metron-writer/metron-writer-common/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java
similarity index 98%
rename from metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java
rename to metron-platform/metron-writer/metron-writer-common/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java
index e94608f..4880ccd 100644
--- a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java
+++ b/metron-platform/metron-writer/metron-writer-common/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java
@@ -33,7 +33,6 @@ import java.util.Collections;
import java.util.List;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.utils.ErrorUtils;
import org.apache.metron.common.writer.BulkMessageWriter;
import org.apache.metron.common.writer.BulkMessage;
import org.apache.metron.common.writer.BulkWriterResponse;
@@ -50,7 +49,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
-@PrepareForTest({BulkWriterComponent.class, ErrorUtils.class})
+@PrepareForTest({BulkWriterComponent.class})
public class BulkWriterComponentTest {
@Rule
@@ -76,7 +75,6 @@ public class BulkWriterComponentTest {
@Before
public void setup() {
MockitoAnnotations.initMocks(this);
- mockStatic(ErrorUtils.class);
message1.put("value", "message1");
message2.put("value", "message2");
messageIds = Arrays.asList(messageId1, messageId2);
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/NoopWriterTest.java b/metron-platform/metron-writer/metron-writer-common/src/test/java/org/apache/metron/writer/NoopWriterTest.java
similarity index 100%
rename from metron-platform/metron-writer/src/test/java/org/apache/metron/writer/NoopWriterTest.java
rename to metron-platform/metron-writer/metron-writer-common/src/test/java/org/apache/metron/writer/NoopWriterTest.java
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/kafka/KafkaWriterTest.java b/metron-platform/metron-writer/metron-writer-common/src/test/java/org/apache/metron/writer/kafka/KafkaWriterTest.java
similarity index 100%
rename from metron-platform/metron-writer/src/test/java/org/apache/metron/writer/kafka/KafkaWriterTest.java
rename to metron-platform/metron-writer/metron-writer-common/src/test/java/org/apache/metron/writer/kafka/KafkaWriterTest.java
diff --git a/metron-platform/metron-writer/src/test/resources/log4j.properties b/metron-platform/metron-writer/metron-writer-common/src/test/resources/log4j.properties
similarity index 100%
rename from metron-platform/metron-writer/src/test/resources/log4j.properties
rename to metron-platform/metron-writer/metron-writer-common/src/test/resources/log4j.properties
diff --git a/metron-platform/metron-writer/metron-writer-storm/pom.xml b/metron-platform/metron-writer/metron-writer-storm/pom.xml
new file mode 100644
index 0000000..5e0ec56
--- /dev/null
+++ b/metron-platform/metron-writer/metron-writer-storm/pom.xml
@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-writer</artifactId>
+ <version>0.7.1</version>
+ </parent>
+ <artifactId>metron-writer-storm</artifactId>
+ <name>metron-writer-storm</name>
+ <description>Components common to Storm</description>
+ <url>https://metron.apache.org/</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ <commons.config.version>1.10</commons.config.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common-storm</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-writer-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${global_storm_version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-hdfs</artifactId>
+ <version>${global_storm_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-test-utilities</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/AckTuplesPolicy.java b/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/AckTuplesPolicy.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/AckTuplesPolicy.java
rename to metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/AckTuplesPolicy.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BatchTimeoutHelper.java b/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/bolt/BatchTimeoutHelper.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BatchTimeoutHelper.java
rename to metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/bolt/BatchTimeoutHelper.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java b/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
rename to metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/ClonedSyncPolicyCreator.java b/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/ClonedSyncPolicyCreator.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/ClonedSyncPolicyCreator.java
rename to metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/ClonedSyncPolicyCreator.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java b/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
rename to metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/PathExtensionFileNameFormat.java b/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/PathExtensionFileNameFormat.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/PathExtensionFileNameFormat.java
rename to metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/PathExtensionFileNameFormat.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java b/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
rename to metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/SourceAwareMoveAction.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java b/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
rename to metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandlerCallback.java b/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/SourceHandlerCallback.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandlerCallback.java
rename to metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/SourceHandlerCallback.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandlerKey.java b/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/SourceHandlerKey.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandlerKey.java
rename to metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/SourceHandlerKey.java
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SyncPolicyCreator.java b/metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/SyncPolicyCreator.java
similarity index 100%
rename from metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SyncPolicyCreator.java
rename to metron-platform/metron-writer/metron-writer-storm/src/main/java/org/apache/metron/writer/hdfs/SyncPolicyCreator.java
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/AckTuplesPolicyTest.java b/metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/AckTuplesPolicyTest.java
similarity index 93%
rename from metron-platform/metron-writer/src/test/java/org/apache/metron/writer/AckTuplesPolicyTest.java
rename to metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/AckTuplesPolicyTest.java
index 04efb3d..f34e186 100644
--- a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/AckTuplesPolicyTest.java
+++ b/metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/AckTuplesPolicyTest.java
@@ -27,6 +27,7 @@ import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import org.json.simple.JSONObject;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
@@ -102,8 +103,8 @@ public class AckTuplesPolicyTest {
ackTuplesPolicy.onFlush(sensorType, response);
- assertEquals(0, ackTuplesPolicy.getTupleMessageMap().size());
- assertEquals(0, ackTuplesPolicy.getTupleErrorMap().size());
+ Assert.assertEquals(0, ackTuplesPolicy.getTupleMessageMap().size());
+ Assert.assertEquals(0, ackTuplesPolicy.getTupleErrorMap().size());
verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM),
new Values(argThat(new MetronErrorJSONMatcher(expectedError1.getJSONObject()))));
verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM),
@@ -155,8 +156,8 @@ public class AckTuplesPolicyTest {
ackTuplesPolicy.onFlush(sensorType, response);
- assertEquals(2, ackTuplesPolicy.getTupleMessageMap().size());
- assertEquals(1, ackTuplesPolicy.getTupleErrorMap().size());
+ Assert.assertEquals(2, ackTuplesPolicy.getTupleMessageMap().size());
+ Assert.assertEquals(1, ackTuplesPolicy.getTupleErrorMap().size());
verify(collector, times(0)).ack(any());
verify(collector, times(0)).reportError(any());
verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM), new Values(argThat(new MetronErrorJSONMatcher(expectedError1.getJSONObject()))));
@@ -167,8 +168,8 @@ public class AckTuplesPolicyTest {
ackTuplesPolicy.onFlush(sensorType, response);
- assertEquals(0, ackTuplesPolicy.getTupleMessageMap().size());
- assertEquals(0, ackTuplesPolicy.getTupleErrorMap().size());
+ Assert.assertEquals(0, ackTuplesPolicy.getTupleMessageMap().size());
+ Assert.assertEquals(0, ackTuplesPolicy.getTupleErrorMap().size());
verify(collector, times(1)).ack(tuple1);
verify(collector, times(1)).ack(tuple2);
verify(collector, times(1)).reportError(e1);
@@ -189,7 +190,7 @@ public class AckTuplesPolicyTest {
ackTuplesPolicy.onFlush(sensorType, response);
- assertEquals(0, ackTuplesPolicy.getTupleMessageMap().size());
+ Assert.assertEquals(0, ackTuplesPolicy.getTupleMessageMap().size());
verify(collector, times(1)).ack(tuple1);
verify(collector, times(1)).ack(tuple2);
verifyNoMoreInteractions(collector);
@@ -211,7 +212,7 @@ public class AckTuplesPolicyTest {
ackTuplesPolicy.onFlush(sensorType, response);
- assertEquals(0, ackTuplesPolicy.getTupleMessageMap().size());
+ Assert.assertEquals(0, ackTuplesPolicy.getTupleMessageMap().size());
verify(collector, times(1)).ack(tuple1);
verifyNoMoreInteractions(collector);
}
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/bolt/BatchTimeoutHelperTest.java b/metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/bolt/BatchTimeoutHelperTest.java
similarity index 100%
rename from metron-platform/metron-writer/src/test/java/org/apache/metron/writer/bolt/BatchTimeoutHelperTest.java
rename to metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/bolt/BatchTimeoutHelperTest.java
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/bolt/BulkMessageWriterBoltTest.java b/metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/bolt/BulkMessageWriterBoltTest.java
similarity index 85%
rename from metron-platform/metron-writer/src/test/java/org/apache/metron/writer/bolt/BulkMessageWriterBoltTest.java
rename to metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/bolt/BulkMessageWriterBoltTest.java
index 0fd784f..3caed2c 100644
--- a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/bolt/BulkMessageWriterBoltTest.java
+++ b/metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/bolt/BulkMessageWriterBoltTest.java
@@ -59,6 +59,7 @@ import org.json.simple.parser.ParseException;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
+import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
@@ -133,8 +134,8 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
.withMessageGetterField("message");
bulkMessageWriterBolt.setCuratorFramework(client);
bulkMessageWriterBolt.setZKCache(cache);
- bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(sensorType,
- new FileInputStream(sampleSensorIndexingConfigPath));
+ bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(BaseEnrichmentBoltTest.sensorType,
+ new FileInputStream("../" + BaseEnrichmentBoltTest.sampleSensorIndexingConfigPath));
// initialize the bolt
bulkMessageWriterBolt.declareOutputFields(declarer);
@@ -151,7 +152,7 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), any());
verify(outputCollector, times(1)).ack(tuple);
verify(outputCollector, times(1)).reportError(any(Throwable.class));
- verifyNoMoreInteractions(outputCollector);
+ Mockito.verifyNoMoreInteractions(outputCollector);
}
@Test
@@ -159,8 +160,8 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
Map stormConf = new HashMap();
bulkMessageWriterBolt.setCuratorFramework(client);
bulkMessageWriterBolt.setZKCache(cache);
- bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(sensorType,
- new FileInputStream(sampleSensorIndexingConfigPath));
+ bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(BaseEnrichmentBoltTest.sensorType,
+ new FileInputStream("../" + BaseEnrichmentBoltTest.sampleSensorIndexingConfigPath));
{
doThrow(new Exception()).when(bulkMessageWriter).init(eq(stormConf), any(WriterConfiguration.class));
try {
@@ -177,20 +178,20 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
{
for(int i = 0; i < 4; i++) {
bulkMessageWriterBolt.execute(tupleList.get(i));
- verify(bulkMessageWriter, times(0)).write(eq(sensorType)
+ verify(bulkMessageWriter, times(0)).write(eq(BaseEnrichmentBoltTest.sensorType)
, any(WriterConfiguration.class), anyList());
}
BulkWriterResponse response = new BulkWriterResponse();
response.addAllSuccesses(messageIdList);
- when(bulkMessageWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(messageList))).thenReturn(response);
+ when(bulkMessageWriter.write(eq(BaseEnrichmentBoltTest.sensorType), any(WriterConfiguration.class), eq(messageList))).thenReturn(response);
bulkMessageWriterBolt.execute(tupleList.get(4));
- verify(bulkMessageWriter, times(1)).write(eq(sensorType)
+ verify(bulkMessageWriter, times(1)).write(eq(BaseEnrichmentBoltTest.sensorType)
, any(WriterConfiguration.class), eq(messageList));
tupleList.forEach(tuple -> verify(outputCollector, times(1)).ack(tuple));
reset(outputCollector);
}
{
- doThrow(new Exception()).when(bulkMessageWriter).write(eq(sensorType), any(WriterConfiguration.class)
+ doThrow(new Exception()).when(bulkMessageWriter).write(eq(BaseEnrichmentBoltTest.sensorType), any(WriterConfiguration.class)
, anyList());
UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.FATAL);
for(int i = 0; i < 5; i++) {
@@ -201,7 +202,7 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
verify(outputCollector, times(5)).emit(eq(Constants.ERROR_STREAM), any(Values.class));
verify(outputCollector, times(1)).reportError(any(Throwable.class));
}
- verifyNoMoreInteractions(outputCollector);
+ Mockito.verifyNoMoreInteractions(outputCollector);
}
@Test
@@ -210,8 +211,8 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
bulkMessageWriterBolt = bulkMessageWriterBolt.withBatchTimeoutDivisor(3);
bulkMessageWriterBolt.setCuratorFramework(client);
bulkMessageWriterBolt.setZKCache(cache);
- bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(sensorType,
- new FileInputStream(sampleSensorIndexingConfigPath));
+ bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(BaseEnrichmentBoltTest.sensorType,
+ new FileInputStream("../" + BaseEnrichmentBoltTest.sampleSensorIndexingConfigPath));
{
bulkMessageWriterBolt.declareOutputFields(declarer);
verify(declarer, times(1)).declareStream(eq("error")
@@ -228,19 +229,19 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
assertEquals(4, batchTimeout);
for(int i = 0; i < 4; i++) {
bulkMessageWriterBolt.execute(tupleList.get(i));
- verify(bulkMessageWriter, times(0)).write(eq(sensorType)
+ verify(bulkMessageWriter, times(0)).write(eq(BaseEnrichmentBoltTest.sensorType)
, any(WriterConfiguration.class), any(List.class));
}
clock.elapseSeconds(5);
BulkWriterResponse response = new BulkWriterResponse();
response.addAllSuccesses(messageIdList);
- when(bulkMessageWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(messageList))).thenReturn(response);
+ when(bulkMessageWriter.write(eq(BaseEnrichmentBoltTest.sensorType), any(WriterConfiguration.class), eq(messageList))).thenReturn(response);
bulkMessageWriterBolt.execute(tupleList.get(4));
- verify(bulkMessageWriter, times(1)).write(eq(sensorType), any(WriterConfiguration.class), eq(messageList));
+ verify(bulkMessageWriter, times(1)).write(eq(BaseEnrichmentBoltTest.sensorType), any(WriterConfiguration.class), eq(messageList));
tupleList.forEach(tuple -> verify(outputCollector, times(1)).ack(tuple));
}
- verifyNoMoreInteractions(outputCollector);
+ Mockito.verifyNoMoreInteractions(outputCollector);
}
@Test
@@ -248,8 +249,8 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
FakeClock clock = new FakeClock();
bulkMessageWriterBolt.setCuratorFramework(client);
bulkMessageWriterBolt.setZKCache(cache);
- bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(sensorType
- , new FileInputStream(sampleSensorIndexingConfigPath));
+ bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(BaseEnrichmentBoltTest.sensorType
+ , new FileInputStream("../" + BaseEnrichmentBoltTest.sampleSensorIndexingConfigPath));
{
bulkMessageWriterBolt.declareOutputFields(declarer);
verify(declarer, times(1)).declareStream(eq("error")
@@ -266,7 +267,7 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
assertEquals(14, batchTimeout);
for(int i = 0; i < 5; i++) {
bulkMessageWriterBolt.execute(tupleList.get(i));
- verify(bulkMessageWriter, times(0)).write(eq(sensorType)
+ verify(bulkMessageWriter, times(0)).write(eq(BaseEnrichmentBoltTest.sensorType)
, any(WriterConfiguration.class), any());
}
Tuple tickTuple = mock(Tuple.class);
@@ -275,23 +276,23 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
when(tickTuple.getSourceStreamId()).thenReturn("__tick"); //mark the tuple as a TickTuple, part 2 of 2
BulkWriterResponse response = new BulkWriterResponse();
response.addAllSuccesses(messageIdList);
- when(bulkMessageWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(messageList))).thenReturn(response);
+ when(bulkMessageWriter.write(eq(BaseEnrichmentBoltTest.sensorType), any(WriterConfiguration.class), eq(messageList))).thenReturn(response);
clock.advanceToSeconds(2);
bulkMessageWriterBolt.execute(tickTuple);
- verify(bulkMessageWriter, times(0)).write(eq(sensorType)
+ verify(bulkMessageWriter, times(0)).write(eq(BaseEnrichmentBoltTest.sensorType)
, any(WriterConfiguration.class)
, eq(messageList));
verify(outputCollector, times(1)).ack(tickTuple); // 1 tick
clock.advanceToSeconds(9);
bulkMessageWriterBolt.execute(tickTuple);
- verify(bulkMessageWriter, times(1)).write(eq(sensorType)
+ verify(bulkMessageWriter, times(1)).write(eq(BaseEnrichmentBoltTest.sensorType)
, any(WriterConfiguration.class)
, eq(messageList));
assertEquals(5, tupleList.size());
tupleList.forEach(tuple -> verify(outputCollector, times(1)).ack(tuple));
verify(outputCollector, times(2)).ack(tickTuple);
}
- verifyNoMoreInteractions(outputCollector);
+ Mockito.verifyNoMoreInteractions(outputCollector);
}
/**
@@ -310,7 +311,8 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
.withMessageGetterField("message");
bolt.setCuratorFramework(client);
bolt.setZKCache(cache);
- bolt.getConfigurations().updateSensorIndexingConfig(sensorType, new FileInputStream(sampleSensorIndexingConfigPath));
+ bolt.getConfigurations().updateSensorIndexingConfig(BaseEnrichmentBoltTest.sensorType,
+ new FileInputStream("../" + BaseEnrichmentBoltTest.sampleSensorIndexingConfigPath));
// initialize the bolt
bolt.declareOutputFields(declarer);
@@ -326,7 +328,7 @@ public class BulkMessageWriterBoltTest extends BaseEnrichmentBoltTest {
verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), any());
verify(outputCollector, times(1)).ack(tuple);
verify(outputCollector, times(1)).reportError(any(Throwable.class));
- verifyNoMoreInteractions(outputCollector);
+ Mockito.verifyNoMoreInteractions(outputCollector);
}
@Test
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/ClonedSyncPolicyCreatorTest.java b/metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/hdfs/ClonedSyncPolicyCreatorTest.java
similarity index 100%
rename from metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/ClonedSyncPolicyCreatorTest.java
rename to metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/hdfs/ClonedSyncPolicyCreatorTest.java
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/HdfsWriterTest.java b/metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/hdfs/HdfsWriterTest.java
similarity index 100%
rename from metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/HdfsWriterTest.java
rename to metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/hdfs/HdfsWriterTest.java
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/PathExtensionFileNameFormatTest.java b/metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/hdfs/PathExtensionFileNameFormatTest.java
similarity index 100%
rename from metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/PathExtensionFileNameFormatTest.java
rename to metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/hdfs/PathExtensionFileNameFormatTest.java
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/SourceHandlerTest.java b/metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/hdfs/SourceHandlerTest.java
similarity index 100%
rename from metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/SourceHandlerTest.java
rename to metron-platform/metron-writer/metron-writer-storm/src/test/java/org/apache/metron/writer/hdfs/SourceHandlerTest.java
diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml
index f8ae4c4..36a4a64 100644
--- a/metron-platform/metron-writer/pom.xml
+++ b/metron-platform/metron-writer/pom.xml
@@ -1,336 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software
- Foundation (ASF) under one or more contributor license agreements. See the
- NOTICE file distributed with this work for additional information regarding
- copyright ownership. The ASF licenses this file to You under the Apache License,
- Version 2.0 (the "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software distributed
- under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
- OR CONDITIONS OF ANY KIND, either express or implied. See the License for
- the specific language governing permissions and limitations under the License.
- -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>metron-writer</artifactId>
+ <packaging>pom</packaging>
+ <name>metron-writer</name>
+ <parent>
+ <artifactId>metron-platform</artifactId>
+ <groupId>org.apache.metron</groupId>
+ <version>0.7.1</version>
+ </parent>
+ <description>Metron parsers and platform support</description>
+ <url>https://metron.apache.org/</url>
+ <scm>
+ <connection>scm:git:https://gitbox.apache.org/repos/asf/metron.git</connection>
+ <developerConnection>scm:git:https://gitbox.apache.org/repos/asf/metron.git</developerConnection>
+ <tag>HEAD</tag>
+ <url>https://gitbox.apache.org/repos/asf/metron.git</url>
+ </scm>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-platform</artifactId>
- <version>0.7.1</version>
- </parent>
- <artifactId>metron-writer</artifactId>
- <name>metron-writer</name>
- <description>Components common to all enrichments</description>
- <url>https://metron.apache.org/</url>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <commons.config.version>1.10</commons.config.version>
- <antlr.version>4.5</antlr.version>
- <guava_version>${global_hbase_guava_version}</guava_version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava_version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- <version>${global_hadoop_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-httpclient</artifactId>
- <groupId>commons-httpclient</groupId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${global_hadoop_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-httpclient</artifactId>
- <groupId>commons-httpclient</groupId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- <version>${global_hbase_version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- </exclusion>
- </exclusions>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${global_hbase_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- <version>${global_storm_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-over-slf4j</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-slf4j-impl</artifactId>
- <groupId>org.apache.logging.log4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>${global_httpclient_version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- <version>${global_kafka_version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jmx</groupId>
- <artifactId>jmxri</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jdmk</groupId>
- <artifactId>jmxtools</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.jms</groupId>
- <artifactId>jms</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-hdfs</artifactId>
- <version>${global_storm_version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${global_hadoop_version}</version>
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- </exclusions>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-common-storm</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment-common</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-test-utilities</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-hbase</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>stellar-common</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.antlr</groupId>
- <artifactId>antlr4-maven-plugin</artifactId>
- <version>${antlr.version}</version>
- <configuration>
- <outputDirectory>${basedir}/src/main/java</outputDirectory>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>antlr4</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${global_shade_version}</version>
- <configuration>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- <artifactSet>
- <excludes>
- <exclude>*slf4j*</exclude>
- </excludes>
- </artifactSet>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>com.google.common</pattern>
- <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern>
- </relocation>
- </relocations>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resources>
- <resource>.yaml</resource>
- <resource>LICENSE.txt</resource>
- <resource>ASL2.0</resource>
- <resource>NOTICE.txt</resource>
- </resources>
- </transformer>
- <!-- UNCOMMENT THIS IF YOU NEED TO REGENERATE THE BEST GUESS NOTICES FILE WHICH REQUIRES PRUNING EVERY RELEASE -->
- <!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
- <addHeader>false</addHeader>
- <projectName>${project.name}</projectName>
- </transformer-->
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass></mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- </build>
-</project>
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+ <modules>
+ <module>metron-writer-common</module>
+ <module>metron-writer-storm</module>
+ </modules>
+</project>
\ No newline at end of file