You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/08/23 12:00:51 UTC
incubator-metron git commit: METRON-368 Simplify Profile
Configuration with Sensible Defaults (nickwallen) closes
apache/incubator-metron#213
Repository: incubator-metron
Updated Branches:
refs/heads/master 3acb21653 -> f5c6d0093
METRON-368 Simplify Profile Configuration with Sensible Defaults (nickwallen) closes apache/incubator-metron#213
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/f5c6d009
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/f5c6d009
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/f5c6d009
Branch: refs/heads/master
Commit: f5c6d00936cc05a34075928bb7027196523c5ae4
Parents: 3acb216
Author: nickwallen <ni...@nickallen.org>
Authored: Tue Aug 23 08:00:18 2016 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Tue Aug 23 08:00:18 2016 -0400
----------------------------------------------------------------------
.../profiler/bolt/ProfileSplitterBolt.java | 3 +-
.../profiler/bolt/ProfileBuilderBoltTest.java | 76 +++++++++++++++++---
.../profiler/bolt/ProfileSplitterBoltTest.java | 55 +++++++++++---
.../src/test/resources/log4j.properties | 28 ++++++++
.../apache/metron/hbase/bolt/HBaseBoltTest.java | 51 +++++--------
.../metron/hbase/client/HBaseClientTest.java | 31 +++++---
.../src/test/resources/log4j.properties | 28 ++++++++
7 files changed, 207 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/f5c6d009/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
index bb19321..5804068 100644
--- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
+++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
@@ -26,6 +26,7 @@ import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
+import org.apache.commons.lang.StringUtils;
import org.apache.metron.common.bolt.ConfiguredProfilerBolt;
import org.apache.metron.common.configuration.profiler.ProfileConfig;
import org.apache.metron.common.configuration.profiler.ProfilerConfig;
@@ -118,7 +119,7 @@ public class ProfileSplitterBolt extends ConfiguredProfilerBolt {
// is this message needed by this profile?
String onlyIf = profile.getOnlyif();
- if (executor.execute(onlyIf, message, Boolean.class)) {
+ if (StringUtils.isBlank(onlyIf) || executor.execute(onlyIf, message, Boolean.class)) {
// what is the name of the entity in this message?
String entity = executor.execute(profile.getForeach(), message, String.class);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/f5c6d009/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java
index c6745f8..5390bd5 100644
--- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java
+++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java
@@ -75,7 +75,7 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
* }
*/
@Multiline
- private String profile;
+ private String basicProfile;
/**
* {
@@ -103,6 +103,28 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
@Multiline
private String profileWithBadInit;
+ /**
+ * {
+ * "profile": "test",
+ * "foreach": "ip_src_addr",
+ * "update": { "x": "2" },
+ * "result": "x"
+ * }
+ */
+ @Multiline
+ private String profileWithNoInit;
+
+ /**
+ * {
+ * "profile": "test",
+ * "foreach": "ip_src_addr",
+ * "init": { "x": "2" },
+ * "result": "x"
+ * }
+ */
+ @Multiline
+ private String profileWithNoUpdate;
+
private JSONObject message;
public static Tuple mockTickTuple() {
@@ -116,8 +138,7 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
return tuple;
}
- @Before
- public void setup() throws Exception {
+ public void setup(String profile) throws Exception {
// parse the input message
JSONParser parser = new JSONParser();
@@ -152,8 +173,9 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
* Ensure that the bolt can update a profile based on new messages that it receives.
*/
@Test
- public void testUpdateProfile() throws Exception {
+ public void testProfileUpdate() throws Exception {
+ setup(basicProfile);
ProfileBuilderBolt bolt = createBolt();
bolt.execute(tuple);
bolt.execute(tuple);
@@ -164,12 +186,46 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
}
/**
+ * If the 'init' field is not defined, then the profile should
+ * behave as normal, but with no variable initialization.
+ */
+ @Test
+ public void testProfileWithNoInit() throws Exception {
+
+ setup(profileWithNoInit);
+ ProfileBuilderBolt bolt = createBolt();
+ bolt.execute(tuple);
+ bolt.execute(tuple);
+
+ // validate
+ assertEquals(2, bolt.getExecutor().getState().get("x"));
+ }
+
+ /**
+ * If the 'update' field is not defined, then no updates should occur as messages
+ * are received.
+ */
+ @Test
+ public void testProfileWithNoUpdate() throws Exception {
+
+ setup(profileWithNoUpdate);
+ ProfileBuilderBolt bolt = createBolt();
+ bolt.execute(tuple);
+ bolt.execute(tuple);
+ bolt.execute(tuple);
+
+ // validate
+ assertEquals(2, bolt.getExecutor().getState().get("x"));
+ }
+
+ /**
* Ensure that the bolt can flush the profile when a tick tuple is received.
*/
@Test
- public void testFlushProfile() throws Exception {
+ public void testProfileFlush() throws Exception {
// setup
+ setup(basicProfile);
ProfileBuilderBolt bolt = createBolt();
bolt.execute(tuple);
bolt.execute(tuple);
@@ -200,8 +256,9 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
* is received from the Splitter.
*/
@Test
- public void testFlushProfileWithNoMessages() throws Exception {
+ public void testProfileFlushWithNoMessages() throws Exception {
+ setup(basicProfile);
ProfileBuilderBolt bolt = createBolt();
// no messages have been received before a flush occurs
@@ -219,6 +276,7 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
@Test
public void testStateClearedAfterFlush() throws Exception {
+ setup(basicProfile);
ProfileBuilderBolt bolt = createBolt();
bolt.execute(tuple);
bolt.execute(tuple);
@@ -236,8 +294,7 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
public void testProfileWithBadUpdate() throws Exception {
// setup - ensure the bad profile is used
- ProfileConfig profileConfig = JSONUtils.INSTANCE.load(profileWithBadUpdate, ProfileConfig.class);
- when(tuple.getValueByField(eq("profile"))).thenReturn(profileConfig);
+ setup(profileWithBadUpdate);
// execute
ProfileBuilderBolt bolt = createBolt();
@@ -255,8 +312,7 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
public void testProfileWithBadInit() throws Exception {
// setup - ensure the bad profile is used
- ProfileConfig profileConfig = JSONUtils.INSTANCE.load(profileWithBadInit, ProfileConfig.class);
- when(tuple.getValueByField(eq("profile"))).thenReturn(profileConfig);
+ setup(profileWithBadInit);
// execute
ProfileBuilderBolt bolt = createBolt();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/f5c6d009/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
index 13f8171..67c1ba3 100644
--- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
+++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
@@ -97,6 +97,23 @@ public class ProfileSplitterBoltTest extends BaseBoltTest {
* {
* "profile": "test",
* "foreach": "ip_src_addr",
+ * "init": {},
+ * "update": {},
+ * "result": 2
+ * }
+ * ]
+ * }
+ */
+ @Multiline
+ private String onlyIfMissing;
+
+ /**
+ * {
+ * "inputTopic": "enrichment",
+ * "profiles": [
+ * {
+ * "profile": "test",
+ * "foreach": "ip_src_addr",
* "onlyif": "NOT-VALID",
* "init": {},
* "update": {},
@@ -106,7 +123,7 @@ public class ProfileSplitterBoltTest extends BaseBoltTest {
* }
*/
@Multiline
- private String invalidOnlyIf;
+ private String onlyIfInvalid;
private JSONObject message;
@@ -137,12 +154,11 @@ public class ProfileSplitterBoltTest extends BaseBoltTest {
}
/**
- * What happens when a message is received that is needed by a profile?
- *
- * This occurs when a profile's 'onlyif' expression is true.
+ * What happens when a profile's 'onlyif' expression is true? The message
+ * should be applied to the profile.
*/
@Test
- public void testMessageNeededByProfile() throws Exception {
+ public void testOnlyIfTrue() throws Exception {
// setup
ProfileSplitterBolt bolt = createBolt(onlyIfTrue);
@@ -158,12 +174,31 @@ public class ProfileSplitterBoltTest extends BaseBoltTest {
}
/**
- * What happens when a message is received that is NOT needed by a profile?
- *
- * This occurs when a profile's 'onlyif' expression is false
+ * All messages are applied to a profile where 'onlyif' is missing. A profile with no
+ * 'onlyif' is treated the same as if 'onlyif=true'.
+ */
+ @Test
+ public void testOnlyIfMissing() throws Exception {
+
+ // setup
+ ProfileSplitterBolt bolt = createBolt(onlyIfMissing);
+
+ // execute
+ bolt.execute(tuple);
+
+ // a tuple should be emitted for the downstream profile builder
+ verify(outputCollector, times(1)).emit(refEq(tuple), any(Values.class));
+
+ // the original tuple should be ack'd
+ verify(outputCollector, times(1)).ack(tuple);
+ }
+
+ /**
+ * What happens when a profile's 'onlyif' expression is false? The message
+ * should NOT be applied to the profile.
*/
@Test
- public void testMessageNotNeededByProfile() throws Exception {
+ public void testOnlyIfFalse() throws Exception {
// setup
ProfileSplitterBolt bolt = createBolt(onlyIfFalse);
@@ -204,7 +239,7 @@ public class ProfileSplitterBoltTest extends BaseBoltTest {
public void testOnlyIfInvalid() throws Exception {
// setup
- ProfileSplitterBolt bolt = createBolt(invalidOnlyIf);
+ ProfileSplitterBolt bolt = createBolt(onlyIfInvalid);
// execute
bolt.execute(tuple);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/f5c6d009/metron-analytics/metron-profiler/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/resources/log4j.properties b/metron-analytics/metron-profiler/src/test/resources/log4j.properties
new file mode 100644
index 0000000..70be8ae
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/test/resources/log4j.properties
@@ -0,0 +1,28 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+# Root logger option
+log4j.rootLogger=ERROR, stdout
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/f5c6d009/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/bolt/HBaseBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/bolt/HBaseBoltTest.java b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/bolt/HBaseBoltTest.java
index d790d51..10b0243 100644
--- a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/bolt/HBaseBoltTest.java
+++ b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/bolt/HBaseBoltTest.java
@@ -22,30 +22,24 @@ package org.apache.metron.hbase.bolt;
import backtype.storm.Constants;
import backtype.storm.tuple.Tuple;
-import backtype.storm.tuple.Values;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.metron.hbase.Widget;
import org.apache.metron.hbase.WidgetMapper;
import org.apache.metron.hbase.client.HBaseClient;
import org.apache.metron.test.bolt.BaseBoltTest;
import org.apache.storm.hbase.bolt.mapper.HBaseMapper;
-import org.junit.After;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Test;
-import org.mockito.Mock;
import java.io.IOException;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
/**
* Tests the HBaseBolt.
@@ -53,23 +47,13 @@ import static org.mockito.Mockito.*;
public class HBaseBoltTest extends BaseBoltTest {
private static final String tableName = "widgets";
-
- @Mock
private HBaseClient client;
-
- @Mock
private Tuple tuple1;
-
- @Mock
private Tuple tuple2;
-
private Widget widget1;
-
private Widget widget2;
-
private HBaseMapper mapper;
-
@Before
public void setupTuples() throws Exception {
@@ -84,19 +68,18 @@ public class HBaseBoltTest extends BaseBoltTest {
@Before
public void setup() throws Exception {
-
- // create a mapper
mapper = new WidgetMapper();
-
+ tuple1 = mock(Tuple.class);
+ tuple2 = mock(Tuple.class);
+ client = mock(HBaseClient.class);
}
/**
* Create a ProfileBuilderBolt to test
*/
- private HBaseBolt createBolt() throws IOException {
+ private HBaseBolt createBolt(int batchSize) throws IOException {
HBaseBolt bolt = new HBaseBolt(tableName, mapper)
- .withBatchSize(2);
-
+ .withBatchSize(batchSize);
bolt.prepare(Collections.emptyMap(), topologyContext, outputCollector);
bolt.setClient(client);
return bolt;
@@ -109,21 +92,23 @@ public class HBaseBoltTest extends BaseBoltTest {
*/
@Test
public void testBatchReady() throws Exception {
- HBaseBolt bolt = createBolt();
+ HBaseBolt bolt = createBolt(2);
bolt.execute(tuple1);
bolt.execute(tuple2);
+
+ // batch size is 2, received 2 tuples - flush the batch
verify(client, times(1)).batchMutate(any(List.class));
}
/**
- * What happens if the batch is not full?
- *
- * If the batch size is 2 and we have only received 2 tuple, the batch should not be flushed.
+ * If the batch size is NOT reached, the batch should NOT be flushed.
*/
@Test
public void testBatchNotReady() throws Exception {
- HBaseBolt bolt = createBolt();
+ HBaseBolt bolt = createBolt(2);
bolt.execute(tuple1);
+
+ // batch size is 2, but only 1 tuple received - do not flush batch
verify(client, times(0)).batchMutate(any(List.class));
}
@@ -132,13 +117,13 @@ public class HBaseBoltTest extends BaseBoltTest {
*/
@Test
public void testTimeFlush() throws Exception {
- HBaseBolt bolt = createBolt();
+ HBaseBolt bolt = createBolt(2);
// the batch is not ready to write
bolt.execute(tuple1);
verify(client, times(0)).batchMutate(any(List.class));
- // the batch should be written after the tick tuple
+ // the batch should be flushed after the tick tuple
bolt.execute(mockTickTuple());
verify(client, times(1)).batchMutate(any(List.class));
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/f5c6d009/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/HBaseClientTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/HBaseClientTest.java b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/HBaseClientTest.java
index f024420..2545266 100644
--- a/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/HBaseClientTest.java
+++ b/metron-platform/metron-hbase/src/test/java/org/apache/metron/hbase/client/HBaseClientTest.java
@@ -22,24 +22,32 @@ package org.apache.metron.hbase.client;
import backtype.storm.tuple.Tuple;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.metron.hbase.TableProvider;
import org.apache.metron.hbase.Widget;
import org.apache.metron.hbase.WidgetMapper;
import org.apache.storm.hbase.bolt.mapper.HBaseMapper;
import org.apache.storm.hbase.bolt.mapper.HBaseProjectionCriteria;
import org.apache.storm.hbase.common.ColumnList;
-import org.junit.*;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
-import java.io.IOException;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
/**
* Tests the HBaseClient
@@ -51,18 +59,19 @@ public class HBaseClientTest {
private static HBaseTestingUtility util;
private HBaseClient client;
private HTableInterface table;
-
private Tuple tuple1;
private Tuple tuple2;
-
private Widget widget1;
private Widget widget2;
-
private HBaseMapper mapper;
@BeforeClass
public static void startHBase() throws Exception {
- util = new HBaseTestingUtility();
+ Configuration config = HBaseConfiguration.create();
+ config.set("hbase.master.hostname", "localhost");
+ config.set("hbase.regionserver.hostname", "localhost");
+
+ util = new HBaseTestingUtility(config);
util.startMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/f5c6d009/metron-platform/metron-hbase/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-hbase/src/test/resources/log4j.properties b/metron-platform/metron-hbase/src/test/resources/log4j.properties
new file mode 100644
index 0000000..70be8ae
--- /dev/null
+++ b/metron-platform/metron-hbase/src/test/resources/log4j.properties
@@ -0,0 +1,28 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+# Root logger option
+log4j.rootLogger=ERROR, stdout
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n