You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/08/22 14:48:29 UTC
[3/3] incubator-metron git commit: METRON-309 Created a Normalcy
Profiler (nickwallen) closes apache/incubator-metron#208
METRON-309 Created a Normalcy Profiler (nickwallen) closes apache/incubator-metron#208
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/81754864
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/81754864
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/81754864
Branch: refs/heads/master
Commit: 817548647efe736dd05a7675d33f12f415ecf46f
Parents: 1ea8d9e
Author: nickwallen <ni...@nickallen.org>
Authored: Mon Aug 22 10:47:24 2016 -0400
Committer: Nick Allen <ni...@nickallen.org>
Committed: Mon Aug 22 10:47:24 2016 -0400
----------------------------------------------------------------------
metron-analytics/metron-profiler/README.md | 250 ++++++++++++++
metron-analytics/metron-profiler/pom.xml | 337 +++++++++++++++++++
.../src/main/assembly/assembly.xml | 72 ++++
.../src/main/config/profiler.properties | 36 ++
.../src/main/flux/profiler/remote.yaml | 114 +++++++
.../metron/profiler/ProfileMeasurement.java | 133 ++++++++
.../profiler/bolt/ProfileBuilderBolt.java | 292 ++++++++++++++++
.../profiler/bolt/ProfileHBaseMapper.java | 136 ++++++++
.../profiler/bolt/ProfileSplitterBolt.java | 151 +++++++++
.../stellar/DefaultStellarExecutor.java | 117 +++++++
.../profiler/stellar/StellarExecutor.java | 64 ++++
.../src/main/scripts/start_profiler_topology.sh | 22 ++
.../zookeeper/readme-example-1/profiler.json | 17 +
.../zookeeper/readme-example-2/profiler.json | 19 ++
.../zookeeper/readme-example-3/profiler.json | 20 ++
.../profiler/bolt/ProfileBuilderBoltTest.java | 269 +++++++++++++++
.../profiler/bolt/ProfileSplitterBoltTest.java | 212 ++++++++++++
.../integration/ConfigUploadComponent.java | 119 +++++++
.../integration/ProfilerIntegrationTest.java | 226 +++++++++++++
.../util/DefaultStellarExecutorTest.java | 161 +++++++++
metron-analytics/pom.xml | 3 +-
.../roles/metron_streaming/defaults/main.yml | 9 +
.../metron_streaming/tasks/copy_bundles.yml | 7 +-
.../roles/metron_streaming/tasks/topologies.yml | 12 +
.../common/bolt/ConfiguredEnrichmentBolt.java | 1 -
.../common/bolt/ConfiguredProfilerBolt.java | 75 +++++
.../common/configuration/ConfigurationType.java | 34 +-
.../configuration/ConfigurationsUtils.java | 105 ++++--
.../configuration/profiler/ProfileConfig.java | 145 ++++++++
.../configuration/profiler/ProfilerConfig.java | 73 ++++
.../profiler/ProfilerConfigurations.java | 53 +++
.../metron/common/dsl/MapVariableResolver.java | 2 +
.../common/stellar/BaseStellarProcessor.java | 1 +
.../metron/enrichment/bolt/HBaseBolt.java | 184 ----------
.../components/ConfigUploadComponent.java | 32 +-
metron-platform/metron-hbase/pom.xml | 128 ++++++-
.../apache/metron/hbase/bolt/BatchHelper.java | 127 +++++++
.../org/apache/metron/hbase/bolt/HBaseBolt.java | 202 +++++++++++
.../apache/metron/hbase/client/HBaseClient.java | 165 +++++++++
.../java/org/apache/metron/hbase/Widget.java | 77 +++++
.../org/apache/metron/hbase/WidgetMapper.java | 55 +++
.../apache/metron/hbase/bolt/HBaseBoltTest.java | 156 +++++++++
.../metron/hbase/client/HBaseClientTest.java | 149 ++++++++
.../components/FluxTopologyComponent.java | 173 +++++-----
.../org/apache/metron/test/mock/MockHTable.java | 5 +-
metron-platform/pom.xml | 1 -
46 files changed, 4418 insertions(+), 323 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/README.md
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/README.md b/metron-analytics/metron-profiler/README.md
new file mode 100644
index 0000000..2ad9c6f
--- /dev/null
+++ b/metron-analytics/metron-profiler/README.md
@@ -0,0 +1,250 @@
+# Metron Profiler
+
+The Profiler is a feature extraction mechanism that can generate a profile describing the behavior of an entity on a network. An entity might be a server, user, subnet or application. Once a profile has been generated defining what normal behavior looks-like, models can be built that identify anomalous behavior.
+
+This is achieved by summarizing the streaming telemetry data consumed by Metron over sliding windows. A summary statistic is applied to the data received within a given window. Collecting this summary across many windows results in a time series that is useful for analysis.
+
+## Usage
+
+Any field contained within a message can be used to generate a profile. A profile can even be produced from combining fields that originate in different data sources. A user has considerable power to transform the data used in a profile by leveraging the Stellar language. A user only need configure the desired profiles in Zookeeper and ensure that the Profiler topology is running.
+
+### Configuration
+
+The Profiler configuration requires a JSON-formatted set of elements, many of which can contain Stellar code. The configuration contains the following elements.
+
+| Name | | Description |
+|--- |--- |--- |
+| profile | Required | A unique name identifying the profile. The field is treated as a string. |
+| foreach | Required | A separate profile is maintained *for each* of these. This is effectively the entity that the profile is describing. The field is expected to contain a Stellar expression whose result is the entity name. For example, if `ip_src_addr` then a separate profile would be maintained for each unique IP source address in the data; 10.0.0.1, 10.0.0.2, etc. |
+| onlyif | Optional | An expression that determines if a message should be applied to the profile. A Stellar expression is expected that when executed returns a boolean. A message is only applied to a profile if this condition is true. This allows a profile to filter the messages that it receives. |
+| init | Optional | A set of expressions that is executed at the start of a window period. A map is expected where the key is the variable name and the value is a Stellar expression. The map can contain 0 or more variables/expressions. At the start of each window period the expression is executed once and stored in a variable with the given name. |
+| update | Required | A set of expressions that is executed when a message is applied to the profile. A map is expected where the key is the variable name and the value is a Stellar expression. The map can include 0 or more variables/expressions. |
+| result | Required | A Stellar expression that is executed when the window period expires. The expression is expected to in some way summarize the messages that were applied to the profile over the window period. The expression must result in a numeric value such as a Double, Long, Float, Short, or Integer. |
+
+#### Examples
+
+Examples of the types of profiles that can be built include the following. Each shows the configuration that would be required to produce the profile. These examples assume a fictitious input messages that looks something like the following.
+
+```
+{
+ "ip_src_addr": "10.0.0.1",
+ "protocol": "HTTPS",
+ "length": "10",
+ "bytes_in": "234"
+},
+{
+ "ip_src_addr": "10.0.0.2",
+ "protocol": "HTTP",
+ "length": "20",
+ "bytes_in": "390"
+},
+{
+ "ip_src_addr": "10.0.0.3",
+ "protocol": "DNS",
+ "length": "30",
+ "bytes_in": "560"
+}
+```
+
+#### Example 1
+
+The total number of bytes of HTTP data for each host. The following configuration would be used to generate this profile.
+
+```
+{
+ "inputTopic": "indexing",
+ "profiles": [
+ {
+ "profile": "example1",
+ "foreach": "ip_src_addr",
+ "onlyif": "protocol == 'HTTP'",
+ "init": {
+ "total_bytes": 0.0
+ },
+ "update": {
+ "total_bytes": "total_bytes + bytes_in"
+ },
+ "result": "total_bytes"
+ }
+ ]
+}
+```
+
+This creates a profile...
+ * Named \u2018example1\u2019
+ * That for each IP source address
+ * Only if the 'protocol' field equals 'HTTP'
+ * Initializes a counter \u2018total_bytes\u2019 to zero
+ * Adds to \u2018total_bytes\u2019 the value of the message's \u2018bytes_in\u2019 field
+ * Returns \u2018total_bytes\u2019 as the result
+
+#### Example 2
+
+The ratio of DNS traffic to HTTP traffic for each host. The following configuration would be used to generate this profile.
+
+```
+{
+ "inputTopic": "indexing",
+ "profiles": [
+ {
+ "profile": "example2",
+ "foreach": "ip_src_addr",
+ "onlyif": "protocol == 'DNS' or protocol == 'HTTP'",
+ "init": {
+ "num_dns": 1.0,
+ "num_http": 1.0
+ },
+ "update": {
+ "num_dns": "num_dns + (if protocol == 'DNS' then 1 else 0)",
+ "num_http": "num_http + (if protocol == 'HTTP' then 1 else 0)"
+ },
+ "result": "num_dns / num_http"
+ }
+ ]
+}
+```
+
+This creates a profile...
+ * Named \u2018example2\u2019
+ * That for each IP source address
+ * Only if the 'protocol' field equals 'HTTP' or 'DNS'
+ * Accumulates the number of DNS requests
+ * Accumulates the number of HTTP requests
+ * Returns the ratio of these as the result
+
+#### Example 3
+
+The average response body length of HTTP traffic. The following configuration would be used to generate this profile.
+
+```
+{
+ "inputTopic": "indexing",
+ "profiles": [
+ {
+ "profile": "example3",
+ "foreach": "ip_src_addr",
+ "onlyif": "protocol == 'HTTP'",
+ "init": {
+ "sum": 0.0,
+ "cnt": 0.0
+ },
+ "update": {
+ "sum": "sum + resp_body_len",
+ "cnt": "cnt + 1"
+ },
+ "result": "sum / cnt"
+ }
+ ]
+}
+```
+
+This creates a profile...
+ * Named \u2018example3\u2019
+ * That for each IP source address
+ * Only if the 'protocol' field equals 'HTTP'
+ * Accumulates the sum of response body length
+ * Accumulates the number of messages
+ * Calculates the average as the result
+
+### Topology Configuration
+
+The Profiler topology also accepts the following configuration settings.
+
+| Setting | Description |
+|--- |--- |
+| profiler.workers | The number of worker processes to create for the topology. |
+| profiler.executors | The number of executors to spawn per component. |
+| profiler.input.topic | The name of the Kafka topic from which to consume data. |
+| profiler.flush.interval.seconds | The duration of a profile's sliding window before it is flushed. |
+| profiler.hbase.salt.divisor | A salt is prepended to the row key to help prevent hotspotting. This constant is used to generate the salt. Ideally, this constant should be roughly equal to the number of nodes in the Hbase cluster. |
+| profiler.hbase.table | The name of the HBase table that profiles are written to. |
+| profiler.hbase.batch | The number of puts that are written in a single batch. |
+| profiler.hbase.flush.interval.seconds | The maximum number of seconds between batch writes to HBase. |
+
+
+## Getting Started
+
+This section will describe the steps required to get your first profile running.
+
+1. Launch the 'Quick Dev' environment.
+ ```
+ $ cd metron-deployment/vagrant/quick-dev-platform/
+ $ ./run.sh
+ ```
+
+2. After the environment has been deployed, then login to the host.
+ ```
+ $ vagrant ssh
+ $ sudo su -
+ $ cd /usr/metron/0.2.0BETA/
+ ```
+
+3. Create a table within HBase that will store the profile data. The table name and column family must match the Profiler topology configuration stored at `/usr/metron/0.2.0BETA/config/profiler.properties`.
+ ```
+ $ /usr/hdp/current/hbase-client/bin/hbase shell
+ hbase(main):001:0> create 'profiler', 'P'
+ ```
+
+4. Shorten the flush intervals to more immediately see results. Edit the Profiler topology properties located at `/usr/metron/0.2.0BETA/config/profiler.properties`. Alter the following two properties.
+ ```
+ profiler.flush.interval.seconds=30
+ profiler.hbase.flush.interval.seconds=30
+ ```
+
+5. Create the Profiler definition in a file located at `/usr/metron/0.2.0BETA/config/zookeeper/profiler.json`. The following JSON will create a profile that simply counts the number of messages.
+ ```
+ {
+ "inputTopic": "indexing",
+ "profiles": [
+ {
+ "profile": "test",
+ "foreach": "ip_src_addr",
+ "onlyif": "true",
+ "init": { "sum": 0 },
+ "update": { "sum": "sum + 1" },
+ "result": "sum"
+ }
+ ]
+ }
+ ```
+
+6. Upload the Profiler definition to Zookeeper.
+ ```
+ $ bin/zk_load_configs.sh -m PUSH -i config/zookeeper/ -z node1:2181
+ ```
+
+7. Start the Profiler topology.
+ ```
+ bin/start_profiler_topology.sh
+ ```
+
+8. Ensure that test messages are being sent to the Profiler's input topic in Kafka. The Profiler will consume messages from the `inputTopic` in the Profiler definition.
+
+9. Check the HBase table to validate that the Profiler is working.
+ ```
+ $ /usr/hdp/current/hbase-client/bin/hbase shell
+ hbase(main):001:0> count 'profiler'
+ ```
+
+## Implementation
+
+The Profiler is implemented as a Storm topology using the following bolts and spouts.
+
+## Topology
+
+### KafkaSpout
+
+A spout that consumes messages from a single Kafka topic. In most cases, the Profiler topology will consume messages from the `indexing` topic. This topic contains fully enriched messages that are ready to be indexed. This ensures that profiles can take advantage of all the available data elements.
+
+### ProfileSplitterBolt
+
+The bolt responsible for filtering incoming messages and directing each to the one or more downstream bolts that are responsible for building a Profile. Each message may be needed by 0, 1 or even many Profiles. Each emitted tuple contains the 'resolved' entity name, the profile definition, and the input message.
+
+### ProfileBuilderBolt
+
+This bolt maintains all of the state required to build a Profile. When the window period expires, the data is summarized as a ProfileMeasurement, all state is flushed, and the ProfileMeasurement is emitted. Each instance of this bolt is responsible for maintaining the state for a single Profile-Entity pair.
+
+### HBaseBolt
+
+A bolt that is responsible for writing to HBase. Most profiles will be flushed every 15 minutes or so. If each ProfileBuilderBolt were responsible for writing to HBase itself, there would be little to no opportunity to optimize these writes. By aggregating the writes from multiple Profile-Entity pairs these writes can be batched, for example.
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml
new file mode 100644
index 0000000..b31df67
--- /dev/null
+++ b/metron-analytics/metron-profiler/pom.xml
@@ -0,0 +1,337 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-analytics</artifactId>
+ <version>0.2.0BETA</version>
+ </parent>
+ <artifactId>metron-profiler</artifactId>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-hbase</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-integration-test</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-test-utilities</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <version>${global_hadoop_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${global_hadoop_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <classifier>tests</classifier>
+ <version>${global_hadoop_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <classifier>tests</classifier>
+ <version>${global_hadoop_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.adrianwalker</groupId>
+ <artifactId>multiline-string</artifactId>
+ <version>0.1.2</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>tools</artifactId>
+ <groupId>sun.jdk</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${global_storm_version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-hdfs</artifactId>
+ <version>${global_storm_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-hbase</artifactId>
+ <version>${global_storm_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.9.2</artifactId>
+ <version>${global_kafka_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${global_hbase_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${global_hbase_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-testing-util</artifactId>
+ <version>${global_hbase_version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${global_mockito_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.tempus-fugit</groupId>
+ <artifactId>tempus-fugit</artifactId>
+ <version>1.2-20140129.191141-5</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <reporting>
+ <plugins>
+ <!-- Normally, dependency report takes time, skip it -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>2.7</version>
+
+ <configuration>
+ <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>emma-maven-plugin</artifactId>
+ <version>1.0-alpha-3</version>
+ <inherited>true</inherited>
+ </plugin>
+ </plugins>
+ </reporting>
+
+ <build>
+ <plugins>
+ <plugin>
+ <!-- Separates the unit tests from the integration tests. -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.12.4</version>
+ <configuration>
+ <!-- Skip the default running of this plug-in (or everything is run twice...see below) -->
+ <argLine>-Xmx2048m</argLine>
+ <skip>true</skip>
+ <!-- Show 100% of the lines from the stack trace (doesn't work) -->
+ <trimStackTrace>false</trimStackTrace>
+ </configuration>
+ <executions>
+ <execution>
+ <id>unit-tests</id>
+ <phase>test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <!-- Never skip running the tests when the test phase is invoked -->
+ <skip>false</skip>
+ <includes>
+ <!-- Include unit tests within integration-test phase. -->
+ <include>**/*Test.java</include>
+ </includes>
+ <excludes>
+ <!-- Exclude integration tests within (unit) test phase. -->
+ <exclude>**/*IntegrationTest.java</exclude>
+ </excludes>
+
+ </configuration>
+ </execution>
+ <execution>
+ <id>integration-tests</id>
+ <phase>integration-test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <!-- Never skip running the tests when the integration-test phase is invoked -->
+ <skip>false</skip>
+ <includes>
+ <!-- Include integration tests within integration-test phase. -->
+ <include>**/*IntegrationTest.java</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${global_shade_version}</version>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>org.apache.metron.guava.metron-profiler</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml.jackson.core</pattern>
+ <shadedPattern>com.fasterxml.jackson.core.metron.elasticsearch</shadedPattern>
+ </relocation>
+ </relocations>
+ <artifactSet>
+ <excludes>
+ <exclude>storm:storm-core:*</exclude>
+ <exclude>storm:storm-lib:*</exclude>
+ <exclude>org.slf4j.impl*</exclude>
+ <exclude>org.slf4j:slf4j-log4j*</exclude>
+ </excludes>
+ </artifactSet>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resource>.yaml</resource>
+ </transformer>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass></mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/assembly/assembly.xml b/metron-analytics/metron-profiler/src/main/assembly/assembly.xml
new file mode 100644
index 0000000..a5f1643
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/main/assembly/assembly.xml
@@ -0,0 +1,72 @@
+<!--
+ ~
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<assembly>
+ <id>archive</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}/src/main/config</directory>
+ <outputDirectory>/config</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0644</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/src/main/scripts</directory>
+ <outputDirectory>/bin</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/src/main/flux</directory>
+ <outputDirectory>/flux</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0644</fileMode>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/target</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ <outputDirectory>/lib</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ </fileSet>
+ </fileSets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/src/main/config/profiler.properties
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/config/profiler.properties b/metron-analytics/metron-profiler/src/main/config/profiler.properties
new file mode 100644
index 0000000..80fd2da
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/main/config/profiler.properties
@@ -0,0 +1,36 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+##### Storm #####
+
+profiler.workers=1
+profiler.executors=0
+profiler.input.topic=indexing
+profiler.flush.interval.seconds=900
+profiler.hbase.salt.divisor=1000
+profiler.hbase.table=profiler
+profiler.hbase.batch=10
+profiler.hbase.flush.interval.seconds=30
+
+##### Kafka #####
+
+kafka.zk=node1:2181
+kafka.broker=node1:6667
+kafka.start=WHERE_I_LEFT_OFF
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
new file mode 100644
index 0000000..d99510a
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
@@ -0,0 +1,114 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "profiler"
+
+config:
+ topology.workers: ${profiler.workers}
+ topology.acker.executors: ${profiler.executors}
+
+components:
+
+ - id: "defaultExecutor"
+ className: "org.apache.metron.profiler.stellar.DefaultStellarExecutor"
+
+ - id: "hbaseMapper"
+ className: "org.apache.metron.profiler.bolt.ProfileHBaseMapper"
+ properties:
+ - name: "saltDivisor"
+ value: ${profiler.hbase.salt.divisor}
+
+ - id: "zkHosts"
+ className: "storm.kafka.ZkHosts"
+ constructorArgs:
+ - "${kafka.zk}"
+
+ - id: "kafkaConfig"
+ className: "org.apache.metron.common.spout.kafka.SpoutConfig"
+ constructorArgs:
+ # zookeeper hosts
+ - ref: "zkHosts"
+ # topic name
+ - "${profiler.input.topic}"
+ # zk root
+ - ""
+ # id
+ - "indexing"
+ configMethods:
+ - name: "from"
+ args:
+ - "${kafka.start}"
+
+spouts:
+
+ - id: "kafkaSpout"
+ className: "storm.kafka.KafkaSpout"
+ constructorArgs:
+ - ref: "kafkaConfig"
+
+bolts:
+
+ - id: "splitterBolt"
+ className: "org.apache.metron.profiler.bolt.ProfileSplitterBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ properties:
+ - name: "executor"
+ ref: "defaultExecutor"
+
+ - id: "builderBolt"
+ className: "org.apache.metron.profiler.bolt.ProfileBuilderBolt"
+ constructorArgs:
+ - "${kafka.zk}"
+ properties:
+ - name: "executor"
+ ref: "defaultExecutor"
+ - name: "flushFrequency"
+ value: ${profiler.flush.interval.seconds}
+
+ - id: "hbaseBolt"
+ className: "org.apache.metron.hbase.bolt.HBaseBolt"
+ constructorArgs:
+ - "${profiler.hbase.table}"
+ - ref: "hbaseMapper"
+ configMethods:
+ - name: "withTableProvider"
+ args: ["${hbase.provider.impl}"]
+ - name: "withBatchSize"
+ args: [${profiler.hbase.batch}]
+ - name: "withFlushIntervalSecs"
+ args: [${profiler.hbase.flush.interval.seconds}]
+
+streams:
+
+ - name: "spout -> splitter"
+ from: "kafkaSpout"
+ to: "splitterBolt"
+ grouping:
+ type: SHUFFLE
+
+ - name: "splitter -> builder"
+ from: "splitterBolt"
+ to: "builderBolt"
+ grouping:
+ type: FIELDS
+ args: ["entity", "profile", "message"]
+
+ - name: "builder -> hbase"
+ from: "builderBolt"
+ to: "hbaseBolt"
+ grouping:
+ type: SHUFFLE
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
new file mode 100644
index 0000000..e240567
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
@@ -0,0 +1,133 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler;
+
+/**
+ * Represents a single data point within a Profile.
+ *
+ * A Profile is effectively a time series. To this end a Profile is composed
+ * of many ProfileMeasurement values which in aggregate form a time series.
+ */
+public class ProfileMeasurement {
+
+ /**
+ * The name of the profile that this measurement is associated with.
+ */
+ private String profileName;
+
+ /**
+ * The name of the entity being profiled.
+ */
+ private String entity;
+
+ /**
+ * When the measurement window was started in milliseconds since the epoch.
+ */
+ private long start;
+
+ /**
+ * When the measurement window closed in milliseconds since the epoch.
+ */
+ private long end;
+
+ /**
+ * The actual measurement itself.
+ */
+ private double value;
+
+ public String getProfileName() {
+ return profileName;
+ }
+
+ public void setProfileName(String profileName) {
+ this.profileName = profileName;
+ }
+
+ public String getEntity() {
+ return entity;
+ }
+
+ public void setEntity(String entity) {
+ this.entity = entity;
+ }
+
+ public long getStart() {
+ return start;
+ }
+
+ public void setStart(long start) {
+ this.start = start;
+ }
+
+ public long getEnd() {
+ return end;
+ }
+
+ public void setEnd(long end) {
+ this.end = end;
+ }
+
+ public double getValue() {
+ return value;
+ }
+
+ public void setValue(double value) {
+ this.value = value;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ ProfileMeasurement that = (ProfileMeasurement) o;
+
+ if (start != that.start) return false;
+ if (end != that.end) return false;
+ if (Double.compare(that.value, value) != 0) return false;
+ if (profileName != null ? !profileName.equals(that.profileName) : that.profileName != null) return false;
+ return entity != null ? entity.equals(that.entity) : that.entity == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result;
+ long temp;
+ result = profileName != null ? profileName.hashCode() : 0;
+ result = 31 * result + (entity != null ? entity.hashCode() : 0);
+ result = 31 * result + (int) (start ^ (start >>> 32));
+ result = 31 * result + (int) (end ^ (end >>> 32));
+ temp = Double.doubleToLongBits(value);
+ result = 31 * result + (int) (temp ^ (temp >>> 32));
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "ProfileMeasurement{" +
+ "profileName='" + profileName + '\'' +
+ ", entity='" + entity + '\'' +
+ ", start=" + start +
+ ", end=" + end +
+ ", value=" + value +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
new file mode 100644
index 0000000..d6656ac
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
@@ -0,0 +1,292 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.bolt;
+
+import backtype.storm.Config;
+import backtype.storm.Constants;
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Values;
+import org.apache.metron.common.bolt.ConfiguredProfilerBolt;
+import org.apache.metron.common.configuration.profiler.ProfileConfig;
+import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.profiler.ProfileMeasurement;
+import org.apache.metron.profiler.stellar.StellarExecutor;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+import static java.lang.String.format;
+
+/**
+ * A bolt that is responsible for building a Profile.
+ *
+ * This bolt maintains the state required to build a Profile. When the window
+ * period expires, the data is summarized as a ProfileMeasurement, all state is
+ * flushed, and the ProfileMeasurement is emitted.
+ *
+ * Each instance of this bolt is responsible for maintaining the state for a single
+ * Profile-Entity pair.
+ */
+public class ProfileBuilderBolt extends ConfiguredProfilerBolt {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(ProfileBuilderBolt.class);
+
+ /**
+ * Executes Stellar code and maintains state across multiple invocations.
+ */
+ private StellarExecutor executor;
+
+ /**
+ * The number of seconds between when the Profile is flushed.
+ */
+ private int flushFrequency;
+
+ /**
+ * A ProfileMeasurement is created and emitted each window period. A Profile
+ * itself is composed of many ProfileMeasurements.
+ */
+ private transient ProfileMeasurement measurement;
+
+ /**
+ * The definition of the Profile that the bolt is building.
+ */
+ private transient ProfileConfig profileConfig;
+
+ /**
+ * Parses JSON messages.
+ */
+ private transient JSONParser parser;
+
+ private OutputCollector collector;
+
+ /**
+ * @param zookeeperUrl The Zookeeper URL that contains the configuration data.
+ */
+ public ProfileBuilderBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
+
+ /**
+ * Defines the frequency at which the bolt will receive tick tuples. Tick tuples are
+ * used to control how often a profile is flushed.
+ */
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ Config conf = new Config();
+ conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, getFlushFrequency());
+ return conf;
+ }
+
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ super.prepare(stormConf, context, collector);
+ this.collector = collector;
+ this.parser = new JSONParser();
+ }
+
+ /**
+ * The builder emits a single field, 'measurement', which contains a ProfileMeasurement. A
+ * ProfileMeasurement is emitted when a time window expires and a flush occurs.
+ */
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ // once the time window expires, a complete ProfileMeasurement is emitted
+ declarer.declare(new Fields("measurement"));
+ }
+
+ @Override
+ public void execute(Tuple input) {
+ try {
+ doExecute(input);
+
+ } catch (Throwable e) {
+ LOG.error(format("Unexpected failure: message='%s', tuple='%s'", e.getMessage(), input), e);
+ collector.reportError(e);
+
+ } finally {
+ collector.ack(input);
+ }
+ }
+
+ /**
+ * Update the execution environment based on data contained in the
+ * message. If the tuple is a tick tuple, then flush the profile
+ * and reset the execution environment.
+ * @param input The tuple to execute.
+ */
+ private void doExecute(Tuple input) {
+
+ if(!isTickTuple(input)) {
+
+ // if this is the first tuple in a window period, initialization is needed
+ if (!isInitialized()) {
+ init(input);
+ }
+
+ // update the profile with data from a new message
+ update(input);
+
+ } else {
+
+ // flush the profile - can only flush if it has been initialized
+ if(isInitialized()) {
+ flush(input);
+ }
+ }
+ }
+
+ /**
+ * Initialize the bolt. Occurs when the first tuple is received at the start
+ * of each window period.
+ * @param input The input tuple
+ */
+ private void init(Tuple input) {
+
+ // save the profile definition - needed later during a flush
+ profileConfig = (ProfileConfig) input.getValueByField("profile");
+
+ // create the measurement which will be saved at the end of the window period
+ measurement = new ProfileMeasurement();
+ measurement.setStart(getTimestamp());
+ measurement.setEntity(input.getStringByField("entity"));
+ measurement.setProfileName(profileConfig.getProfile());
+
+ // execute the 'init' expression
+ try {
+ JSONObject message = (JSONObject) input.getValueByField("message");
+ Map<String, String> expressions = profileConfig.getInit();
+ expressions.forEach((var, expr) -> executor.assign(var, expr, message));
+
+ } catch(ParseException e) {
+ String msg = format("Bad 'init' expression: %s, profile=%s, entity=%s, start=%d",
+ e.getMessage(), measurement.getProfileName(), measurement.getEntity(), measurement.getStart());
+ throw new ParseException(msg, e);
+ }
+ }
+
+ /**
+ * Update the Profile based on data contained in a new message.
+ * @param input The tuple containing a new message.
+ */
+ private void update(Tuple input) {
+ JSONObject message = (JSONObject) input.getValueByField("message");
+
+ // execute each of the 'update' expressions
+ try {
+ Map<String, String> expressions = profileConfig.getUpdate();
+ expressions.forEach((var, expr) -> executor.assign(var, expr, message));
+
+ } catch(ParseException e) {
+ String msg = format("Bad 'update' expression: %s, profile=%s, entity=%s, start=%d",
+ e.getMessage(), measurement.getProfileName(), measurement.getEntity(), measurement.getStart());
+ throw new ParseException(msg, e);
+ }
+ }
+
+ /**
+ * Flush the Profile.
+ *
+ * Executed on a fixed time period when a tick tuple is received. Completes
+ * and emits the ProfileMeasurement. Clears all state in preparation for
+ * the next window period.
+ */
+ private void flush(Tuple tickTuple) {
+ LOG.info(String.format("Flushing profile: profile=%s, entity=%s, start=%d",
+ measurement.getProfileName(), measurement.getEntity(), measurement.getStart()));
+
+ // execute the 'result' expression
+ Double result;
+ try {
+ String resultExpr = profileConfig.getResult();
+ result = executor.execute(resultExpr, new JSONObject(), Double.class);
+ } catch(ParseException e) {
+ throw new ParseException("Bad 'result' expression", e);
+ }
+
+ // emit the completed profile measurement
+ measurement.setEnd(getTimestamp());
+ measurement.setValue(result);
+ emit(measurement, tickTuple);
+
+ // clear the execution state to prepare for the next window
+ executor.clearState();
+
+ // reset measurement - used as a flag to indicate if initialized
+ measurement = null;
+ }
+
+ /**
+ * Emits a message containing a ProfileMeasurement.
+ * @param measurement The completed ProfileMeasurement.
+ */
+ private void emit(ProfileMeasurement measurement, Tuple anchor) {
+ collector.emit(anchor, new Values(measurement));
+ collector.ack(anchor);
+ }
+
+ /**
+ * Returns a value that can be used as the current timestamp. Allows subclasses
+ * to override, if necessary.
+ */
+ private long getTimestamp() {
+ return System.currentTimeMillis();
+ }
+
+ /**
+ * Has the Stellar execution environment already been initialized
+ * @return True, it it has been initialized.
+ */
+ private boolean isInitialized() {
+ return measurement != null;
+ }
+
+ /**
+ * Is this a tick tuple?
+ * @param tuple The tuple
+ */
+ protected static boolean isTickTuple(Tuple tuple) {
+ return Constants.SYSTEM_COMPONENT_ID.equals(tuple.getSourceComponent()) &&
+ Constants.SYSTEM_TICK_STREAM_ID.equals(tuple.getSourceStreamId());
+ }
+
+ public StellarExecutor getExecutor() {
+ return executor;
+ }
+
+ public void setExecutor(StellarExecutor executor) {
+ this.executor = executor;
+ }
+
+ public int getFlushFrequency() {
+ return flushFrequency;
+ }
+
+ public void setFlushFrequency(int flushFrequency) {
+ this.flushFrequency = flushFrequency;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java
new file mode 100644
index 0000000..1c515be
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileHBaseMapper.java
@@ -0,0 +1,136 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.bolt;
+
+import backtype.storm.tuple.Tuple;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.profiler.ProfileMeasurement;
+import org.apache.storm.hbase.bolt.mapper.HBaseMapper;
+import org.apache.storm.hbase.common.ColumnList;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Calendar;
+
+/**
+ * An HbaseMapper that defines how a ProfileMeasurement is persisted within an HBase table.
+ */
+public class ProfileHBaseMapper implements HBaseMapper {
+
+ /**
+ * A salt is prepended to the row key to help prevent hotspotting. This constant is used
+ * to generate the salt. Ideally, this constant should be roughly equal to the number of
+ * nodes in the Hbase cluster.
+ */
+ private int saltDivisor = 10;
+
+ /**
+ * The name of the column family.
+ */
+ private String columnFamily;
+
+ public ProfileHBaseMapper() {
+ setColumnFamily("P");
+ }
+
+ /**
+ * Define the row key for a ProfileMeasurement.
+ * @param tuple The tuple containing a ProfileMeasurement.
+ * @return The Hbase row key.
+ */
+ @Override
+ public byte[] rowKey(Tuple tuple) {
+ ProfileMeasurement m = (ProfileMeasurement) tuple.getValueByField("measurement");
+
+ // create a calendar to determine day-of-week, etc
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTimeInMillis(m.getStart());
+
+ return Bytes.toBytes(getSalt(m.getStart()) +
+ m.getProfileName() +
+ calendar.get(Calendar.DAY_OF_WEEK) +
+ calendar.get(Calendar.WEEK_OF_MONTH) +
+ calendar.get(Calendar.MONTH) +
+ calendar.get(Calendar.YEAR) +
+ m.getEntity() +
+ m.getStart());
+ }
+
+ /**
+ * Defines how the fields within a ProfileMeasurement are mapped to HBase.
+ * @param tuple The tuple containing the ProfileMeasurement.
+ */
+ @Override
+ public ColumnList columns(Tuple tuple) {
+ ProfileMeasurement measurement = (ProfileMeasurement) tuple.getValueByField("measurement");
+
+ byte[] cfBytes = Bytes.toBytes(columnFamily);
+ ColumnList cols = new ColumnList();
+ cols.addColumn(cfBytes, QPROFILE, Bytes.toBytes(measurement.getProfileName()));
+ cols.addColumn(cfBytes, QENTITY, Bytes.toBytes(measurement.getEntity()));
+ cols.addColumn(cfBytes, QSTART, Bytes.toBytes(measurement.getStart()));
+ cols.addColumn(cfBytes, QEND, Bytes.toBytes(measurement.getEnd()));
+ cols.addColumn(cfBytes, QVALUE, Bytes.toBytes(measurement.getValue()));
+
+ return cols;
+ }
+
+ /**
+ * Calculates a salt value that is used as part of the row key.
+ *
+ * The salt is calculated as 'md5(timestamp) % N' where N is a configurable value that ideally
+ * is close to the number of nodes in the Hbase cluster.
+ *
+ * @param epoch The timestamp in epoch millis to use in generating the salt.
+ */
+ private int getSalt(long epoch) {
+ try {
+ MessageDigest digest = MessageDigest.getInstance("MD5");
+ byte[] hash = digest.digest(Bytes.toBytes(epoch));
+ return Bytes.toInt(hash) % saltDivisor;
+
+ } catch(NoSuchAlgorithmException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String getColumnFamily() {
+ return columnFamily;
+ }
+
+ public void setColumnFamily(String columnFamily) {
+ this.columnFamily = columnFamily;
+ }
+
+ public int getSaltDivisor() {
+ return saltDivisor;
+ }
+
+ public void setSaltDivisor(int saltDivisor) {
+ this.saltDivisor = saltDivisor;
+ }
+
+ public static final byte[] QPROFILE = Bytes.toBytes("profile");
+ public static final byte[] QENTITY = Bytes.toBytes("entity");
+ public static final byte[] QSTART = Bytes.toBytes("start");
+ public static final byte[] QEND = Bytes.toBytes("end");
+ public static final byte[] QVALUE = Bytes.toBytes("value");
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
new file mode 100644
index 0000000..bb19321
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
@@ -0,0 +1,151 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.bolt;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Values;
+import org.apache.metron.common.bolt.ConfiguredProfilerBolt;
+import org.apache.metron.common.configuration.profiler.ProfileConfig;
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
+import org.apache.metron.profiler.stellar.StellarExecutor;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Map;
+
+import static java.lang.String.format;
+
+/**
+ * The bolt responsible for filtering incoming messages and directing
+ * each to the one or more bolts responsible for building a Profile. Each
+ * message may be needed by 0, 1 or even many Profiles.
+ */
+public class ProfileSplitterBolt extends ConfiguredProfilerBolt {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(ProfileSplitterBolt.class);
+
+ private OutputCollector collector;
+
+ /**
+ * JSON parser.
+ */
+ private transient JSONParser parser;
+
+ /**
+ * Executes Stellar code.
+ */
+ private StellarExecutor executor;
+
+ /**
+ * @param zookeeperUrl The Zookeeper URL that contains the configuration for this bolt.
+ */
+ public ProfileSplitterBolt(String zookeeperUrl) {
+ super(zookeeperUrl);
+ }
+
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ super.prepare(stormConf, context, collector);
+ this.collector = collector;
+ this.parser = new JSONParser();
+ }
+
+ @Override
+ public void execute(Tuple input) {
+ try {
+ doExecute(input);
+
+ } catch (IllegalArgumentException | ParseException | UnsupportedEncodingException e) {
+ LOG.error(format("Unexpected failure: message='%s', tuple='%s'", e.getMessage(), input), e);
+ collector.reportError(e);
+
+ } finally {
+ collector.ack(input);
+ }
+ }
+
+ private void doExecute(Tuple input) throws ParseException, UnsupportedEncodingException {
+
+ // retrieve the input message
+ byte[] data = input.getBinary(0);
+ JSONObject message = (JSONObject) parser.parse(new String(data, "UTF8"));
+
+ // ensure there is a valid profiler configuration
+ ProfilerConfig config = getProfilerConfig();
+ if(config == null) {
+ throw new IllegalArgumentException("Fatal: Unable to find valid profiler definition");
+ }
+
+ // apply the message to each of the profile definitions
+ for (ProfileConfig profile: config.getProfiles()) {
+ applyProfile(profile, input, message);
+ }
+ }
+
+ /**
+ * Applies a message to a Profile definition.
+ * @param profile The profile definition.
+ * @param input The input tuple that delivered the message.
+ * @param message The message that may be needed by the profile.
+ */
+ private void applyProfile(ProfileConfig profile, Tuple input, JSONObject message) throws ParseException, UnsupportedEncodingException {
+
+ // is this message needed by this profile?
+ String onlyIf = profile.getOnlyif();
+ if (executor.execute(onlyIf, message, Boolean.class)) {
+
+ // what is the name of the entity in this message?
+ String entity = executor.execute(profile.getForeach(), message, String.class);
+
+ // emit a message for the bolt responsible for building this profile
+ collector.emit(input, new Values(entity, profile, message));
+ }
+ }
+
+ /**
+ * Each emitted tuple contains the following fields.
+ * <p><ol>
+ * <li> entity - The name of the entity. The actual result of executing the Stellar expression.
+ * <li> profile - The profile definition that the message needs applied to.
+ * <li> message - The message containing JSON-formatted data that needs applied to a profile.
+ * </ol></p>
+ */
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("entity", "profile", "message"));
+ }
+
+ public StellarExecutor getExecutor() {
+ return executor;
+ }
+
+ public void setExecutor(StellarExecutor executor) {
+ this.executor = executor;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java
new file mode 100644
index 0000000..d608222
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java
@@ -0,0 +1,117 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.stellar;
+
+import org.apache.metron.common.dsl.MapVariableResolver;
+import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.common.dsl.VariableResolver;
+import org.apache.metron.common.stellar.StellarPredicateProcessor;
+import org.apache.metron.common.stellar.StellarProcessor;
+import org.apache.metron.common.utils.ConversionUtils;
+import org.json.simple.JSONObject;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * The default implementation of a StellarExecutor.
+ */
+public class DefaultStellarExecutor implements StellarExecutor, Serializable {
+
+ /**
+ * The current state of the Stellar execution environment.
+ */
+ private Map<String, Object> state;
+
+ public DefaultStellarExecutor() {
+ clearState();
+ }
+
+ /**
+ * @param initialState Initial state loaded into the execution environment.
+ */
+ public DefaultStellarExecutor(Map<String, Object> initialState) {
+ this.state = new HashMap<>(initialState);
+ }
+
+ @Override
+ public Map<String, Object> getState() {
+ return new HashMap<>(state);
+ }
+
+ /**
+ * Execute an expression and assign the result to a variable.
+ *
+ * @param variable The variable name to assign to.
+ * @param expression The expression to execute.
+ * @param message The message that provides additional context for the expression.
+ */
+ @Override
+ public void assign(String variable, String expression, JSONObject message) {
+ Object result = execute(expression, message);
+ state.put(variable, result);
+ }
+
+ /**
+ * Execute a Stellar expression and returns the result.
+ *
+ * @param expr The expression to execute.
+ * @param message The message that is accessible when Stellar is executed.
+ * @param clazz The expected class of the expression's result.
+ * @param <T> The expected class of the expression's result.
+ */
+ @Override
+ public <T> T execute(String expr, JSONObject message, Class<T> clazz) {
+ Object resultObject = execute(expr, message);
+
+ // perform type conversion, if necessary
+ T result = ConversionUtils.convert(resultObject, clazz);
+ if(result == null) {
+ throw new IllegalArgumentException(String.format("Unexpected type: expected=%s, actual=%s, expression=%s",
+ clazz.getSimpleName(), resultObject.getClass().getSimpleName(), expr));
+ }
+
+ return result;
+ }
+
+ @Override
+ public void clearState() {
+ this.state = new HashMap<>();
+ }
+
+ /**
+ * Execute a Stellar expression.
+ *
+ * @param expr The expression to execute.
+ * @param msg The message that is accessible when Stellar is executed.
+ */
+ private Object execute(String expr, JSONObject msg) {
+ try {
+ VariableResolver resolver = new MapVariableResolver(state, msg);
+ StellarProcessor processor = new StellarProcessor();
+ return processor.parse(expr, resolver);
+
+ } catch (ParseException e) {
+ throw new ParseException(String.format("Bad expression: expr=%s, msg=%s, state=%s", expr, msg, state));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/stellar/StellarExecutor.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/stellar/StellarExecutor.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/stellar/StellarExecutor.java
new file mode 100644
index 0000000..9005e00
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/stellar/StellarExecutor.java
@@ -0,0 +1,64 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.stellar;
+
+import org.json.simple.JSONObject;
+
+import java.util.Map;
+
+/**
+ * Executes Stellar expressions and maintains state across multiple invocations.
+ *
+ * There are two sets of functions in Stellar currently. One can be executed with
+ * a PredicateProcessor and the other a TransformationProcessor. This interface
+ * abstracts away that complication.
+ */
+public interface StellarExecutor {
+
+ /**
+ * Execute an expression and assign the result to a variable.
+ *
+ * @param variable The name of the variable to assign to.
+ * @param expression The expression to execute.
+ * @param message The message that provides additional context for the expression.
+ */
+ void assign(String variable, String expression, JSONObject message);
+
+ /**
+ * Execute a Stellar expression and return the result.
+ *
+ * @param expression The expression to execute.
+ * @param message The message that is accessible when Stellar is executed.
+ * @param clazz The expected class of the expression's result.
+ * @param <T> The expected class of the expression's result.
+ */
+ <T> T execute(String expression, JSONObject message, Class<T> clazz);
+
+ /**
+ * The current state of the Stellar execution environment.
+ */
+ Map<String, Object> getState();
+
+ /**
+ * Removes all state from the execution environment.
+ */
+ void clearState();
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/src/main/scripts/start_profiler_topology.sh
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/scripts/start_profiler_topology.sh b/metron-analytics/metron-profiler/src/main/scripts/start_profiler_topology.sh
new file mode 100644
index 0000000..6f78471
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/main/scripts/start_profiler_topology.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+METRON_VERSION=${project.version}
+METRON_HOME=/usr/metron/$METRON_VERSION
+TOPOLOGY_JAR=${project.artifactId}-$METRON_VERSION.jar
+storm jar $METRON_HOME/lib/$TOPOLOGY_JAR org.apache.storm.flux.Flux --remote $METRON_HOME/flux/profiler/remote.yaml --filter $METRON_HOME/config/profiler.properties
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-1/profiler.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-1/profiler.json b/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-1/profiler.json
new file mode 100644
index 0000000..f811df7
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-1/profiler.json
@@ -0,0 +1,17 @@
+{
+ "inputTopic": "indexing",
+ "profiles": [
+ {
+ "profile": "example1",
+ "foreach": "ip_src_addr",
+ "onlyif": "protocol == 'HTTP'",
+ "init": {
+ "total_bytes": 0.0
+ },
+ "update": {
+ "total_bytes": "total_bytes + bytes_in"
+ },
+ "result": "total_bytes"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-2/profiler.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-2/profiler.json b/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-2/profiler.json
new file mode 100644
index 0000000..2c6a278
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-2/profiler.json
@@ -0,0 +1,19 @@
+{
+ "inputTopic": "indexing",
+ "profiles": [
+ {
+ "profile": "example2",
+ "foreach": "ip_src_addr",
+ "onlyif": "protocol == 'DNS' or protocol == 'HTTP'",
+ "init": {
+ "num_dns": 1.0,
+ "num_http": 1.0
+ },
+ "update": {
+ "num_dns": "num_dns + (if protocol == 'DNS' then 1 else 0)",
+ "num_http": "num_http + (if protocol == 'HTTP' then 1 else 0)"
+ },
+ "result": "num_dns / num_http"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-3/profiler.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-3/profiler.json b/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-3/profiler.json
new file mode 100644
index 0000000..96c5935
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-3/profiler.json
@@ -0,0 +1,20 @@
+{
+ "inputTopic": "indexing",
+ "profiles": [
+ {
+ "profile": "example3",
+ "foreach": "ip_src_addr",
+ "onlyif": "protocol == 'HTTP'",
+ "init": {
+ "sum": 0.0,
+ "cnt": 0.0
+ },
+ "update": {
+ "sum": "sum + resp_body_len",
+ "cnt": "cnt + 1"
+ },
+ "result": "sum / cnt"
+ }
+ ]
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/81754864/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java
new file mode 100644
index 0000000..1528e31
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java
@@ -0,0 +1,269 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.bolt;
+
+import backtype.storm.Constants;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Values;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.configuration.profiler.ProfileConfig;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.profiler.ProfileMeasurement;
+import org.apache.metron.profiler.stellar.DefaultStellarExecutor;
+import org.apache.metron.test.bolt.BaseBoltTest;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.*;
+
+/**
+ * Tests the ProfileBuilderBolt.
+ */
+public class ProfileBuilderBoltTest extends BaseBoltTest {
+
+ /**
+ * {
+ * "ip_src_addr": "10.0.0.1",
+ * "ip_dst_addr": "10.0.0.20"
+ * }
+ */
+ @Multiline
+ private String input;
+
+ /**
+ * {
+ * "profile": "test",
+ * "foreach": "ip_src_addr",
+ * "onlyif": "true",
+ * "init": {
+ * "x": 10,
+ * "y": 20
+ * },
+ * "update": {
+ * "x": "x + 10",
+ * "y": "y + 20"
+ * },
+ * "result": "x + y"
+ * }
+ */
+ @Multiline
+ private String profile;
+
+ /**
+ * {
+ * "profile": "test",
+ * "foreach": "ip_src_addr",
+ * "onlyif": "true",
+ * "init": { "x": 10 },
+ * "update": { "x": "x + 'string'" },
+ * "result": "x"
+ * }
+ */
+ @Multiline
+ private String profileWithBadUpdate;
+
+ /**
+ * {
+ * "profile": "test",
+ * "foreach": "ip_src_addr",
+ * "onlyif": "true",
+ * "init": { "x": "10 + 'string'" },
+ * "update": { "x": "x + 2" },
+ * "result": "x"
+ * }
+ */
+ @Multiline
+ private String profileWithBadInit;
+
+ private JSONObject message;
+
+ public static Tuple mockTickTuple() {
+ return mockTuple(Constants.SYSTEM_COMPONENT_ID, Constants.SYSTEM_TICK_STREAM_ID);
+ }
+
+ public static Tuple mockTuple(String componentId, String streamId) {
+ Tuple tuple = mock(Tuple.class);
+ when(tuple.getSourceComponent()).thenReturn(componentId);
+ when(tuple.getSourceStreamId()).thenReturn(streamId);
+ return tuple;
+ }
+
+ @Before
+ public void setup() throws Exception {
+
+ // parse the input message
+ JSONParser parser = new JSONParser();
+ message = (JSONObject) parser.parse(input);
+
+ // the tuple will contain the original message
+ when(tuple.getValueByField(eq("message"))).thenReturn(message);
+
+ // the tuple will contain the 'fully resolved' name of the entity
+ when(tuple.getStringByField(eq("entity"))).thenReturn("10.0.0.1");
+
+ // the tuple will contain the profile definition
+ ProfileConfig profileConfig = JSONUtils.INSTANCE.load(profile, ProfileConfig.class);
+ when(tuple.getValueByField(eq("profile"))).thenReturn(profileConfig);
+ }
+
+ /**
+ * Create a ProfileBuilderBolt to test
+ */
+ private ProfileBuilderBolt createBolt() throws IOException {
+
+ ProfileBuilderBolt bolt = new ProfileBuilderBolt("zookeeperURL");
+ bolt.setCuratorFramework(client);
+ bolt.setTreeCache(cache);
+ bolt.setExecutor(new DefaultStellarExecutor());
+
+ bolt.prepare(new HashMap<>(), topologyContext, outputCollector);
+ return bolt;
+ }
+
+ /**
+ * Ensure that the bolt can update a profile based on new messages that it receives.
+ */
+ @Test
+ public void testUpdateProfile() throws Exception {
+
+ ProfileBuilderBolt bolt = createBolt();
+ bolt.execute(tuple);
+ bolt.execute(tuple);
+
+ // validate that x=10+10+10 y=20+20+20
+ assertEquals(10+10+10.0, bolt.getExecutor().getState().get("x"));
+ assertEquals(20+20+20.0, bolt.getExecutor().getState().get("y"));
+ }
+
+ /**
+ * Ensure that the bolt can flush the profile when a tick tuple is received.
+ */
+ @Test
+ public void testFlushProfile() throws Exception {
+
+ // setup
+ ProfileBuilderBolt bolt = createBolt();
+ bolt.execute(tuple);
+ bolt.execute(tuple);
+
+ // execute - the tick tuple triggers a flush of the profile
+ bolt.execute(mockTickTuple());
+
+ // capture the ProfileMeasurement that should be emitted
+ ArgumentCaptor<Values> arg = ArgumentCaptor.forClass(Values.class);
+ verify(outputCollector, times(1)).emit(refEq(tuple), arg.capture());
+
+ Values actual = arg.getValue();
+ ProfileMeasurement measurement = (ProfileMeasurement) actual.get(0);
+
+ // verify
+ assertThat(measurement.getValue(), equalTo(90.0));
+ assertThat(measurement.getEntity(), equalTo("10.0.0.1"));
+ assertThat(measurement.getProfileName(), equalTo("test"));
+ }
+
+ /**
+ * What happens if we try to flush, but have yet to receive any messages to
+ * apply to the profile?
+ *
+ * The ProfileBuilderBolt will not have received the data necessary from the
+ * ProfileSplitterBolt, like the entity and profile name, that is required
+ * to perform the flush. The flush has to be skipped until this information
+ * is received from the Splitter.
+ */
+ @Test
+ public void testFlushProfileWithNoMessages() throws Exception {
+
+ ProfileBuilderBolt bolt = createBolt();
+
+ // no messages have been received before a flush occurs
+ bolt.execute(mockTickTuple());
+ bolt.execute(mockTickTuple());
+ bolt.execute(mockTickTuple());
+
+ // no ProfileMeasurement should be written to the ProfileStore
+ verify(outputCollector, times(0)).emit(any(Values.class));
+ }
+
+ /**
+ * The executor's state should be cleared after a flush.
+ */
+ @Test
+ public void testStateClearedAfterFlush() throws Exception {
+
+ ProfileBuilderBolt bolt = createBolt();
+ bolt.execute(tuple);
+ bolt.execute(tuple);
+
+ // execute - should clear state from previous tuples
+ bolt.execute(mockTickTuple());
+
+ assertThat(bolt.getExecutor().getState().size(), equalTo(0));
+ }
+
+ /**
+ * What happens when the profile contains a bad Stellar expression?
+ */
+ @Test
+ public void testProfileWithBadUpdate() throws Exception {
+
+ // setup - ensure the bad profile is used
+ ProfileConfig profileConfig = JSONUtils.INSTANCE.load(profileWithBadUpdate, ProfileConfig.class);
+ when(tuple.getValueByField(eq("profile"))).thenReturn(profileConfig);
+
+ // execute
+ ProfileBuilderBolt bolt = createBolt();
+ bolt.execute(tuple);
+
+ // verify - expect the tuple to be acked and an error reported
+ verify(outputCollector, times(1)).ack(eq(tuple));
+ verify(outputCollector, times(1)).reportError(any());
+ }
+
+ /**
+ * What happens when the profile contains a bad Stellar expression?
+ */
+ @Test
+ public void testProfileWithBadInit() throws Exception {
+
+ // setup - ensure the bad profile is used
+ ProfileConfig profileConfig = JSONUtils.INSTANCE.load(profileWithBadInit, ProfileConfig.class);
+ when(tuple.getValueByField(eq("profile"))).thenReturn(profileConfig);
+
+ // execute
+ ProfileBuilderBolt bolt = createBolt();
+ bolt.execute(tuple);
+
+ // verify - expect the tuple to be acked and an error reported
+ verify(outputCollector, times(1)).ack(eq(tuple));
+ verify(outputCollector, times(1)).reportError(any());
+ }
+}