You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2016/11/22 19:09:45 UTC

incubator-metron git commit: METRON-575 State from different profiles can be co-mingled incorrectly (nickwallen) closes apache/incubator-metron#362

Repository: incubator-metron
Updated Branches:
  refs/heads/master f4e1065b6 -> 5b3667f89


METRON-575 State from different profiles can be co-mingled incorrectly (nickwallen) closes apache/incubator-metron#362


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/5b3667f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/5b3667f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/5b3667f8

Branch: refs/heads/master
Commit: 5b3667f899cbaaa8b758db177d31ec02dffef0d5
Parents: f4e1065
Author: nickwallen <ni...@nickallen.org>
Authored: Tue Nov 22 14:08:47 2016 -0500
Committer: Nick Allen <ni...@nickallen.org>
Committed: Tue Nov 22 14:08:47 2016 -0500

----------------------------------------------------------------------
 metron-analytics/metron-profiler/README.md      |   5 +-
 .../src/main/config/profiler.properties         |   2 +
 .../src/main/flux/profiler/remote.yaml          |   5 +-
 .../profiler/bolt/ProfileBuilderBolt.java       | 299 +++++++++++++------
 .../metron/profiler/bolt/ProfileState.java      | 101 +++++++
 .../zookeeper/readme-example-2/profiler.json    |   8 +-
 .../zookeeper/write-integer/profiler.json       |   1 -
 .../profiler/bolt/ProfileBuilderBoltTest.java   |  42 +--
 .../integration/ProfilerIntegrationTest.java    |  70 +++--
 9 files changed, 391 insertions(+), 142 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5b3667f8/metron-analytics/metron-profiler/README.md
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/README.md b/metron-analytics/metron-profiler/README.md
index 3a52133..6a853a1 100644
--- a/metron-analytics/metron-profiler/README.md
+++ b/metron-analytics/metron-profiler/README.md
@@ -161,13 +161,16 @@ The Profiler runs as an independent Storm topology.  The configuration for the P
 | profiler.executors | The number of executors to spawn per component.  |
 | profiler.input.topic | The name of the Kafka topic from which to consume data.  |
 | profiler.period.duration | The duration of each profile period.  This value should be defined along with `profiler.period.duration.units`.  |
-| profiler.period.duration.units | The units used to specify the profile period duration.  This value should be defined along with `profiler.period.duration`. |
+| profiler.period.duration.units | The units used to specify the `profiler.period.duration`. |
+| profiler.ttl | If a message has not been applied to a Profile in this period of time, the Profile will be forgotten and its resources will be cleaned up. This value should be defined along with `profiler.ttl.units`. |
+| profiler.ttl.units | The units used to specify the `profiler.ttl`. |
 | profiler.hbase.salt.divisor  |  A salt is prepended to the row key to help prevent hotspotting.  This constant is used to generate the salt.  Ideally, this constant should be roughly equal to the number of nodes in the Hbase cluster.  |
 | profiler.hbase.table | The name of the HBase table that profiles are written to.  |
 | profiler.hbase.column.family | The column family used to store profiles. |
 | profiler.hbase.batch | The number of puts that are written in a single batch.  |
 | profiler.hbase.flush.interval.seconds | The maximum number of seconds between batch writes to HBase. |
 
+
 After altering the configuration, start the Profiler.
 
 ```

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5b3667f8/metron-analytics/metron-profiler/src/main/config/profiler.properties
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/config/profiler.properties b/metron-analytics/metron-profiler/src/main/config/profiler.properties
index a3448aa..1f4cc0d 100644
--- a/metron-analytics/metron-profiler/src/main/config/profiler.properties
+++ b/metron-analytics/metron-profiler/src/main/config/profiler.properties
@@ -25,6 +25,8 @@ profiler.executors=0
 profiler.input.topic=indexing
 profiler.period.duration=15
 profiler.period.duration.units=MINUTES
+profiler.ttl=30
+profiler.ttl.units=MINUTES
 profiler.hbase.salt.divisor=1000
 profiler.hbase.table=profiler
 profiler.hbase.column.family=P

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5b3667f8/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
index c818a70..c6cee0c 100644
--- a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
+++ b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
@@ -89,12 +89,11 @@ bolts:
         className: "org.apache.metron.profiler.bolt.ProfileBuilderBolt"
         constructorArgs:
             - "${kafka.zk}"
-        properties:
-            - name: "executor"
-              ref: "defaultExecutor"
         configMethods:
             - name: "withPeriodDuration"
               args: [${profiler.period.duration}, "${profiler.period.duration.units}"]
+            - name: "withTimeToLive"
+              args: [${profiler.ttl}, "${profiler.ttl.units}"]
 
     -   id: "hbaseBolt"
         className: "org.apache.metron.hbase.bolt.HBaseBolt"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5b3667f8/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
index 6e26bd0..f1bef74 100644
--- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
+++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java
@@ -20,7 +20,17 @@
 
 package org.apache.metron.profiler.bolt;
 
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
 import org.apache.commons.beanutils.BeanMap;
+import org.apache.metron.common.bolt.ConfiguredProfilerBolt;
+import org.apache.metron.common.configuration.profiler.ProfileConfig;
+import org.apache.metron.common.dsl.Context;
+import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.common.dsl.StellarFunctions;
+import org.apache.metron.profiler.ProfileMeasurement;
+import org.apache.metron.profiler.stellar.DefaultStellarExecutor;
+import org.apache.metron.profiler.stellar.StellarExecutor;
 import org.apache.storm.Config;
 import org.apache.storm.Constants;
 import org.apache.storm.task.OutputCollector;
@@ -29,13 +39,6 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
 import org.apache.storm.tuple.Fields;
 import org.apache.storm.tuple.Tuple;
 import org.apache.storm.tuple.Values;
-import org.apache.metron.common.bolt.ConfiguredProfilerBolt;
-import org.apache.metron.common.configuration.profiler.ProfileConfig;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.StellarFunctions;
-import org.apache.metron.profiler.ProfileMeasurement;
-import org.apache.metron.profiler.stellar.StellarExecutor;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
 import org.slf4j.Logger;
@@ -46,6 +49,7 @@ import java.util.HashMap;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
 import static java.lang.String.format;
@@ -58,17 +62,12 @@ import static org.apache.commons.collections.CollectionUtils.isEmpty;
  * period expires, the data is summarized as a ProfileMeasurement, all state is
  * flushed, and the ProfileMeasurement is emitted.
  *
- * Each instance of this bolt is responsible for maintaining the state for a single
- * Profile-Entity pair.
  */
 public class ProfileBuilderBolt extends ConfiguredProfilerBolt {
 
   protected static final Logger LOG = LoggerFactory.getLogger(ProfileBuilderBolt.class);
 
-  /**
-   * Executes Stellar code and maintains state across multiple invocations.
-   */
-  private StellarExecutor executor;
+  private OutputCollector collector;
 
   /**
    * The duration of each profile period in milliseconds.
@@ -76,23 +75,23 @@ public class ProfileBuilderBolt extends ConfiguredProfilerBolt {
   private long periodDurationMillis;
 
   /**
-   * A ProfileMeasurement is created and emitted each window period.  A Profile
-   * itself is composed of many ProfileMeasurements.
+   * If a message has not been applied to a Profile in this number of milliseconds,
+   * the Profile will be forgotten and its resources will be cleaned up.
+   *
+   * The TTL must be at least greater than the period duration.
    */
-  private transient ProfileMeasurement measurement;
+  private long timeToLiveMillis;
 
   /**
-   * The definition of the Profile that the bolt is building.
+   * Maintains the state of a profile which is unique to a profile/entity pair.
    */
-  private transient ProfileConfig profileConfig;
+  private transient Cache<String, ProfileState> profileCache;
 
   /**
    * Parses JSON messages.
    */
   private transient JSONParser parser;
 
-  private OutputCollector collector;
-
   /**
    * @param zookeeperUrl The Zookeeper URL that contains the configuration data.
    */
@@ -112,21 +111,22 @@ public class ProfileBuilderBolt extends ConfiguredProfilerBolt {
     return conf;
   }
 
-  protected void initializeStellar() {
-    Context context = new Context.Builder()
-            .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client)
-            .with(Context.Capabilities.GLOBAL_CONFIG, () -> getConfigurations().getGlobalConfig())
-            .build();
-    StellarFunctions.initialize(context);
-    executor.setContext(context);
-  }
-
   @Override
   public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
     super.prepare(stormConf, context, collector);
+
+    if(timeToLiveMillis < periodDurationMillis) {
+      String msg = String.format("invalid configuration: expect profile TTL (%d) greater than period duration (%d)",
+              timeToLiveMillis, periodDurationMillis);
+      throw new IllegalStateException(msg);
+    }
+
     this.collector = collector;
     this.parser = new JSONParser();
-    initializeStellar();
+    this.profileCache = CacheBuilder
+            .newBuilder()
+            .expireAfterAccess(timeToLiveMillis, TimeUnit.MILLISECONDS)
+            .build();
   }
 
   /**
@@ -159,18 +159,16 @@ public class ProfileBuilderBolt extends ConfiguredProfilerBolt {
    * and reset the execution environment.
    * @param input The tuple to execute.
    */
-  private void doExecute(Tuple input) {
+  private void doExecute(Tuple input) throws ExecutionException {
 
-    if(!isTickTuple(input)) {
-      if (!isInitialized()) {
-        init(input);
-      }
-      update(input);
+    if(isTickTuple(input)) {
+      flush(input);
 
     } else {
-      if(isInitialized()) {
-        flush(input);
+      if (!isInitialized(input)) {
+        init(input);
       }
+      update(input);
     }
   }
 
@@ -179,26 +177,22 @@ public class ProfileBuilderBolt extends ConfiguredProfilerBolt {
    * of each window period.
    * @param input The input tuple
    */
-  private void init(Tuple input) {
+  private void init(Tuple input) throws ExecutionException {
 
-    // save the profile definition - needed later during a flush
-    profileConfig = (ProfileConfig) input.getValueByField("profile");
+    ProfileState state = getProfileState(input);
+    try {
 
-    // create the measurement which will be saved at the end of the window period
-    measurement = new ProfileMeasurement(
-            profileConfig.getProfile(),
-            input.getStringByField("entity"),
-            getTimestamp(),
-            periodDurationMillis,
-            TimeUnit.MILLISECONDS);
+      // the original telemetry message is provided as additional context for the 'update' expressions
+      JSONObject message = getMessage(input);
 
-    // execute the 'init' expression
-    try {
-      JSONObject message = (JSONObject) input.getValueByField("message");
-      Map<String, String> expressions = profileConfig.getInit();
-      expressions.forEach((var, expr) -> executor.assign(var, expr, message));
+      // execute the 'init' expression
+      Map<String, String> expressions = state.getDefinition().getInit();
+      expressions.forEach((var, expr) -> state.getExecutor().assign(var, expr, message));
 
     } catch(ParseException e) {
+
+      // make it brilliantly clear that one of the 'init' expressions is bad
+      ProfileMeasurement measurement = state.getMeasurement();
       String msg = format("Bad 'init' expression: %s, profile=%s, entity=%s",
               e.getMessage(), measurement.getProfileName(), measurement.getEntity());
       throw new ParseException(msg, e);
@@ -209,15 +203,22 @@ public class ProfileBuilderBolt extends ConfiguredProfilerBolt {
    * Update the Profile based on data contained in a new message.
    * @param input The tuple containing a new message.
    */
-  private void update(Tuple input) {
-    JSONObject message = (JSONObject) input.getValueByField("message");
+  private void update(Tuple input) throws ExecutionException {
 
-    // execute each of the 'update' expressions
+    ProfileState state = getProfileState(input);
     try {
-      Map<String, String> expressions = profileConfig.getUpdate();
-      expressions.forEach((var, expr) -> executor.assign(var, expr, message));
+
+      // the original telemetry message is provided as additional context for the 'update' expressions
+      JSONObject message = getMessage(input);
+
+      // execute each of the 'update' expressions
+      Map<String, String> expressions = state.getDefinition().getUpdate();
+      expressions.forEach((var, expr) -> state.getExecutor().assign(var, expr, message));
 
     } catch(ParseException e) {
+
+      // make it brilliantly clear that one of the 'update' expressions is bad
+      ProfileMeasurement measurement = state.getMeasurement();
       String msg = format("Bad 'update' expression: %s, profile=%s, entity=%s",
               e.getMessage(), measurement.getProfileName(), measurement.getEntity());
       throw new ParseException(msg, e);
@@ -225,49 +226,92 @@ public class ProfileBuilderBolt extends ConfiguredProfilerBolt {
   }
 
   /**
-   * Flush the Profile.
+   * Flush the Profiles.
    *
    * Executed on a fixed time period when a tick tuple is received.  Completes
    * and emits the ProfileMeasurement.  Clears all state in preparation for
    * the next window period.
    */
   private void flush(Tuple tickTuple) {
-    LOG.info(String.format("Flushing profile: profile=%s, entity=%s",
-            measurement.getProfileName(), measurement.getEntity()));
 
-    // calculate the result
-    Object result = executeResult(profileConfig.getResult());
-    measurement.setValue(result);
+    // flush each of the profiles maintain by this bolt
+    profileCache.asMap().forEach((key, profileState) -> {
 
-    // calculate the groups
-    List<Object> groups = executeGroupBy(profileConfig.getGroupBy());
-    measurement.setGroups(groups);
+      ProfileMeasurement measurement = profileState.getMeasurement();
+      StellarExecutor executor = profileState.getExecutor();
+      ProfileConfig definition = profileState.getDefinition();
+      LOG.info(String.format("Flushing profile: profile=%s, entity=%s", measurement.getProfileName(), measurement.getEntity()));
 
-    // emit the completed profile measurement
-    emit(measurement, tickTuple);
+      // execute the 'result' and 'group by' expressions
+      Object value = executeResult(definition.getResult(), executor, measurement);
+      measurement.setValue(value);
 
-    // Execute the update with the old state
-    Map<String, String> tickUpdate = profileConfig.getTickUpdate();
-    Map<String, Object> state = executor.getState();
-    if(tickUpdate != null) {
-      tickUpdate.forEach((var, expr) -> executor.assign(var, expr, Collections.singletonMap("result", result)));
-    }
-    // clear the execution state to prepare for the next window
-    executor.clearState();
-    //make sure that we bring along the update state
-    if(tickUpdate != null) {
-      tickUpdate.forEach((var, expr) -> executor.getState().put(var, state.get(var)));
-    }
+      List<Object> groups = executeGroupBy(definition.getGroupBy(), executor, measurement);
+      measurement.setGroups(groups);
+
+      // emit the completed profile measurement
+      emit(measurement, definition);
+
+      // execute the update with the old state
+      Map<String, String> tickUpdate = definition.getTickUpdate();
+      Map<String, Object> state = executor.getState();
+      if(tickUpdate != null) {
+        tickUpdate.forEach((var, expr) -> executor.assign(var, expr, Collections.singletonMap("result", value)));
+      }
 
-    // reset measurement - used as a flag to indicate if initialized
-    measurement = null;
+      // clear the execution state to prepare for the next window
+      executor.clearState();
+
+      //make sure that we bring along the update state
+      if(tickUpdate != null) {
+        tickUpdate.forEach((var, expr) -> executor.getState().put(var, state.get(var)));
+      }
+
+      // cache maintenance
+      profileCache.cleanUp();
+    });
+  }
+
+  /**
+   * Create the state necessary to build a Profile.
+   * @param tuple The tuple that needs applied to a profile.
+   */
+  private ProfileState createProfileState(Tuple tuple) {
+
+    // extract the profile definition
+    ProfileConfig profileDefinition = getProfileDefinition(tuple);
+
+    // create the profile measurement which will be emitted at the end of the window period
+    ProfileMeasurement measurement = new ProfileMeasurement(
+            profileDefinition.getProfile(),
+            getEntity(tuple),
+            getTimestamp(),
+            periodDurationMillis,
+            TimeUnit.MILLISECONDS);
+
+    // create the executor
+    StellarExecutor executor = new DefaultStellarExecutor();
+    Context context = new Context.Builder()
+            .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client)
+            .with(Context.Capabilities.GLOBAL_CONFIG, () -> getConfigurations().getGlobalConfig())
+            .build();
+    StellarFunctions.initialize(context);
+    executor.setContext(context);
+
+    // create the profile state which is maintained within a cache for a fixed period of time
+    ProfileState state = new ProfileState();
+    state.setExecutor(executor);
+    state.setDefinition(profileDefinition);
+    state.setMeasurement(measurement);
+
+    return state;
   }
 
   /**
    * Executes the 'result' expression of a Profile.
    * @return The result of evaluating the 'result' expression.
    */
-  private Object executeResult(String expression) {
+  private Object executeResult(String expression, StellarExecutor executor, ProfileMeasurement measurement) {
     Object result;
     try {
       result = executor.execute(expression, new JSONObject(), Object.class);
@@ -288,7 +332,7 @@ public class ProfileBuilderBolt extends ConfiguredProfilerBolt {
    * @param expressions The 'groupBy' expressions to execute.
    * @return The result of executing the 'groupBy' expressions.
    */
-  private List<Object> executeGroupBy(List<String> expressions) {
+  private List<Object> executeGroupBy(List<String> expressions, StellarExecutor executor, ProfileMeasurement measurement) {
     List<Object> groups = new ArrayList<>();
 
     if(!isEmpty(expressions)) {
@@ -314,10 +358,66 @@ public class ProfileBuilderBolt extends ConfiguredProfilerBolt {
   /**
    * Emits a message containing a ProfileMeasurement and the Profile configuration.
    * @param measurement The completed ProfileMeasurement.
-   * @param anchor The original tuple to be used as the anchor.
+   * @param definition The profile definition.
+   */
+  private void emit(ProfileMeasurement measurement, ProfileConfig definition) {
+    collector.emit(new Values(measurement, definition));
+  }
+
+  /**
+   * Builds the key that is used to lookup the ProfileState within the cache.
+   * @param tuple A tuple.
    */
-  private void emit(ProfileMeasurement measurement, Tuple anchor) {
-    collector.emit(anchor, new Values(measurement, profileConfig));
+  private String cacheKey(Tuple tuple) {
+    return String.format("%s:%s", getProfileDefinition(tuple).getProfile(), getEntity(tuple));
+  }
+
+  /**
+   * Retrieves the state associated with a Profile.  If none exists, the state will
+   * be initialized.
+   * @param tuple The tuple.
+   */
+  protected ProfileState getProfileState(Tuple tuple) throws ExecutionException {
+    return profileCache.get(cacheKey(tuple), () -> createProfileState(tuple));
+  }
+
+  /**
+   * Extracts the profile definition from a tuple.
+   * @param tuple The tuple sent by the splitter bolt.
+   */
+  private ProfileConfig getProfileDefinition(Tuple tuple) {
+    ProfileConfig definition = (ProfileConfig) tuple.getValueByField("profile");
+    if(definition == null) {
+      throw new IllegalStateException("invalid tuple received: missing profile definition");
+    }
+
+    return definition;
+  }
+
+  /**
+   * Extracts the name of the entity from a tuple.
+   * @param tuple The tuple sent by the splitter bolt.
+   */
+  private String getEntity(Tuple tuple) {
+    String entity = tuple.getStringByField("entity");
+    if(entity == null) {
+      throw new IllegalStateException("invalid tuple received: missing entity name");
+    }
+
+    return entity;
+  }
+
+  /**
+   * Extracts the original telemetry message from a tuple.
+   * @param input The tuple sent by the splitter bolt.
+   */
+  private JSONObject getMessage(Tuple input) {
+    JSONObject message = (JSONObject) input.getValueByField("message");
+    if(message == null) {
+      throw new IllegalStateException("invalid tuple received: missing message");
+    }
+
+    return message;
   }
 
   /**
@@ -332,8 +432,8 @@ public class ProfileBuilderBolt extends ConfiguredProfilerBolt {
    * Has the Stellar execution environment already been initialized
    * @return True, it it has been initialized.
    */
-  private boolean isInitialized() {
-    return measurement != null;
+  private boolean isInitialized(Tuple tuple) {
+    return profileCache.getIfPresent(cacheKey(tuple)) != null;
   }
 
   /**
@@ -345,19 +445,22 @@ public class ProfileBuilderBolt extends ConfiguredProfilerBolt {
             Constants.SYSTEM_TICK_STREAM_ID.equals(tuple.getSourceStreamId());
   }
 
-  public StellarExecutor getExecutor() {
-    return executor;
+  public ProfileBuilderBolt withPeriodDurationMillis(long periodDurationMillis) {
+    this.periodDurationMillis = periodDurationMillis;
+    return this;
   }
 
-  public void setExecutor(StellarExecutor executor) {
-    this.executor = executor;
+  public ProfileBuilderBolt withPeriodDuration(int duration, TimeUnit units) {
+    return withPeriodDurationMillis(units.toMillis(duration));
   }
 
-  public void setPeriodDurationMillis(long periodDurationMillis) {
-    this.periodDurationMillis = periodDurationMillis;
+  public ProfileBuilderBolt withTimeToLiveMillis(long timeToLiveMillis) {
+    this.timeToLiveMillis = timeToLiveMillis;
+    return this;
   }
 
-  public void withPeriodDuration(int duration, TimeUnit units) {
-    setPeriodDurationMillis(units.toMillis(duration));
+  public ProfileBuilderBolt withTimeToLive(int duration, TimeUnit units) {
+    return withTimeToLiveMillis(units.toMillis(duration));
   }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5b3667f8/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileState.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileState.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileState.java
new file mode 100644
index 0000000..f6c48b4
--- /dev/null
+++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileState.java
@@ -0,0 +1,101 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.bolt;
+
+import org.apache.metron.common.configuration.profiler.ProfileConfig;
+import org.apache.metron.profiler.ProfileMeasurement;
+import org.apache.metron.profiler.stellar.StellarExecutor;
+
+/**
+ * The state that must be maintained for each [profile, entity] pair when building a Profile.
+ */
+public class ProfileState {
+
+  /**
+   * A ProfileMeasurement is created and emitted each window period.  A Profile
+   * itself is composed of many ProfileMeasurements.
+   */
+  private ProfileMeasurement measurement;
+
+  /**
+   * The definition of the Profile that the bolt is building.
+   */
+  private ProfileConfig definition;
+
+  /**
+   * Executes Stellar code and maintains state across multiple invocations.
+   */
+  private StellarExecutor executor;
+
+  public ProfileMeasurement getMeasurement() {
+    return measurement;
+  }
+
+  public void setMeasurement(ProfileMeasurement measurement) {
+    this.measurement = measurement;
+  }
+
+  public ProfileConfig getDefinition() {
+    return definition;
+  }
+
+  public void setDefinition(ProfileConfig definition) {
+    this.definition = definition;
+  }
+
+  public StellarExecutor getExecutor() {
+    return executor;
+  }
+
+  public void setExecutor(StellarExecutor executor) {
+    this.executor = executor;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    ProfileState that = (ProfileState) o;
+
+    if (measurement != null ? !measurement.equals(that.measurement) : that.measurement != null) return false;
+    if (definition != null ? !definition.equals(that.definition) : that.definition != null) return false;
+    return executor != null ? executor.equals(that.executor) : that.executor == null;
+
+  }
+
+  @Override
+  public int hashCode() {
+    int result = measurement != null ? measurement.hashCode() : 0;
+    result = 31 * result + (definition != null ? definition.hashCode() : 0);
+    result = 31 * result + (executor != null ? executor.hashCode() : 0);
+    return result;
+  }
+
+  @Override
+  public String toString() {
+    return "ProfileState{" +
+            "measurement=" + measurement +
+            ", definition=" + definition +
+            ", executor=" + executor +
+            '}';
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5b3667f8/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-2/profiler.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-2/profiler.json b/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-2/profiler.json
index 1a8a1b7..e5d8f39 100644
--- a/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-2/profiler.json
+++ b/metron-analytics/metron-profiler/src/test/config/zookeeper/readme-example-2/profiler.json
@@ -5,12 +5,12 @@
       "foreach": "ip_src_addr",
       "onlyif": "protocol == 'DNS' or protocol == 'HTTP'",
       "init": {
-        "num_dns": 0.0,
-        "num_http": 0.0
+        "num_dns": 1.0,
+        "num_http": 1.0
       },
       "update": {
-        "num_dns": "if protocol == 'DNS' then num_dns + 1 else num_dns",
-        "num_http": "if protocol == 'HTTP' then num_http + 1 else num_http"
+        "num_dns": "num_dns + (if protocol == 'DNS' then 1 else 0)",
+        "num_http": "num_http + (if protocol == 'HTTP' then 1 else 0)"
       },
       "result": "num_dns / num_http"
     }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5b3667f8/metron-analytics/metron-profiler/src/test/config/zookeeper/write-integer/profiler.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/config/zookeeper/write-integer/profiler.json b/metron-analytics/metron-profiler/src/test/config/zookeeper/write-integer/profiler.json
index bf78365..8f24cea 100644
--- a/metron-analytics/metron-profiler/src/test/config/zookeeper/write-integer/profiler.json
+++ b/metron-analytics/metron-profiler/src/test/config/zookeeper/write-integer/profiler.json
@@ -3,7 +3,6 @@
     {
       "profile": "example1",
       "foreach": "ip_src_addr",
-      "onlyif": "true",
       "init": {},
       "update": {},
       "result": "TO_INTEGER(10.0)"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5b3667f8/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java
index 42a5cdf..43149e2 100644
--- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java
+++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileBuilderBoltTest.java
@@ -20,17 +20,16 @@
 
 package org.apache.metron.profiler.bolt;
 
-import org.apache.log4j.Level;
-import org.apache.metron.test.utils.UnitTestHelper;
-import org.apache.storm.Constants;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.tuple.Values;
 import org.adrianwalker.multilinestring.Multiline;
+import org.apache.log4j.Level;
 import org.apache.metron.common.configuration.profiler.ProfileConfig;
 import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.profiler.ProfileMeasurement;
-import org.apache.metron.profiler.stellar.DefaultStellarExecutor;
 import org.apache.metron.test.bolt.BaseBoltTest;
+import org.apache.metron.test.utils.UnitTestHelper;
+import org.apache.storm.Constants;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
 import org.junit.Test;
@@ -44,7 +43,12 @@ import static org.hamcrest.CoreMatchers.equalTo;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
 import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.refEq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 /**
  * Tests the ProfileBuilderBolt.
@@ -98,10 +102,10 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
     ProfileBuilderBolt bolt = new ProfileBuilderBolt("zookeeperURL");
     bolt.setCuratorFramework(client);
     bolt.setTreeCache(cache);
-    bolt.setExecutor(new DefaultStellarExecutor());
-    bolt.setPeriodDurationMillis(TimeUnit.MINUTES.toMillis(15));
-
+    bolt.withPeriodDuration(10, TimeUnit.MINUTES);
+    bolt.withTimeToLive(30, TimeUnit.MINUTES);
     bolt.prepare(new HashMap<>(), topologyContext, outputCollector);
+
     return bolt;
   }
 
@@ -136,8 +140,9 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
     bolt.execute(tuple);
 
     // validate that x=10+10+10 y=20+20+20
-    assertEquals(10+10+10.0, bolt.getExecutor().getState().get("x"));
-    assertEquals(20+20+20.0, bolt.getExecutor().getState().get("y"));
+    ProfileState state = bolt.getProfileState(tuple);
+    assertEquals(10+10+10.0, state.getExecutor().getState().get("x"));
+    assertEquals(20+20+20.0, state.getExecutor().getState().get("y"));
   }
 
   /**
@@ -164,7 +169,8 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
     bolt.execute(tuple);
 
     // validate
-    assertEquals(2, bolt.getExecutor().getState().get("x"));
+    ProfileState state = bolt.getProfileState(tuple);
+    assertEquals(2, state.getExecutor().getState().get("x"));
   }
 
   /**
@@ -192,7 +198,8 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
     bolt.execute(tuple);
 
     // validate
-    assertEquals(2, bolt.getExecutor().getState().get("x"));
+    ProfileState state = bolt.getProfileState(tuple);
+    assertEquals(2, state.getExecutor().getState().get("x"));
   }
 
   /**
@@ -212,7 +219,7 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
 
     // capture the ProfileMeasurement that should be emitted
     ArgumentCaptor<Values> arg = ArgumentCaptor.forClass(Values.class);
-    verify(outputCollector, times(1)).emit(refEq(tuple), arg.capture());
+    verify(outputCollector, times(1)).emit(arg.capture());
 
     Values actual = arg.getValue();
     ProfileMeasurement measurement = (ProfileMeasurement) actual.get(0);
@@ -261,7 +268,8 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
     // execute - should clear state from previous tuples
     bolt.execute(mockTickTuple());
 
-    assertThat(bolt.getExecutor().getState().size(), equalTo(0));
+    ProfileState state = bolt.getProfileState(tuple);
+    assertThat(state.getExecutor().getState().size(), equalTo(0));
   }
 
   /**
@@ -359,7 +367,7 @@ public class ProfileBuilderBoltTest extends BaseBoltTest {
 
     // capture the ProfileMeasurement that should be emitted
     ArgumentCaptor<Values> arg = ArgumentCaptor.forClass(Values.class);
-    verify(outputCollector, times(1)).emit(refEq(tuple), arg.capture());
+    verify(outputCollector, times(1)).emit(arg.capture());
 
     Values actual = arg.getValue();
     ProfileMeasurement measurement = (ProfileMeasurement) actual.get(0);

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5b3667f8/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java
index ffca5ec..fc1c24f 100644
--- a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java
+++ b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java
@@ -21,6 +21,7 @@
 package org.apache.metron.profiler.integration;
 
 import org.adrianwalker.multilinestring.Multiline;
+import org.apache.commons.math.util.MathUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Result;
@@ -45,6 +46,7 @@ import org.junit.Test;
 import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -137,8 +139,12 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
     waitOrTimeout(() -> profilerTable.getPutLog().size() > 0,
             timeout(seconds(90)));
 
+    // verify - only 10.0.0.2 sends 'HTTP', thus there should be only 1 value
+    List<Double> actuals = read(columnBuilder.getColumnQualifier("value"), Double.class);
+    Assert.assertEquals(1, actuals.size());
+
     // verify - there are 5 'HTTP' each with 390 bytes
-    double actual = read(columnBuilder.getColumnQualifier("value"), Double.class);
+    double actual = actuals.get(0);
     Assert.assertEquals(390.0 * 5, actual, 0.01);
   }
 
@@ -154,13 +160,22 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
     fluxComponent.submitTopology();
     kafkaComponent.writeMessages(Constants.INDEXING_TOPIC, input);
 
+    // expect 2 values written by the profile; one for 10.0.0.2 and another for 10.0.0.3
+    final int expected = 2;
+
     // verify - ensure the profile is being persisted
-    waitOrTimeout(() -> profilerTable.getPutLog().size() > 0,
+    waitOrTimeout(() -> profilerTable.getPutLog().size() >= expected,
             timeout(seconds(90)));
 
-    // verify - there are 5 'HTTP' and 5 'DNS' messages thus 5/5 = 1
-    double actual = read(columnBuilder.getColumnQualifier("value"), Double.class);
-    Assert.assertEquals(5.0 / 5.0, actual, 0.01);
+    // verify - expect 2 results as 2 hosts involved; 10.0.0.2 sends 'HTTP' and 10.0.0.3 send 'DNS'
+    List<Double> actuals = read(columnBuilder.getColumnQualifier("value"), Double.class);
+    Assert.assertEquals(expected, actuals.size());
+
+    // verify - 10.0.0.3 -> 1/6
+    Assert.assertTrue(actuals.stream().anyMatch(val -> MathUtils.equals(val, 1.0/6.0, 0.1)));
+
+    // verify - 10.0.0.2 -> 6/1
+    Assert.assertTrue(actuals.stream().anyMatch(val -> MathUtils.equals(val, 6.0/1.0, 0.1)));
   }
 
   /**
@@ -179,8 +194,12 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
     waitOrTimeout(() -> profilerTable.getPutLog().size() > 0,
             timeout(seconds(90)));
 
+    // verify - only 10.0.0.2 sends 'HTTP', thus there should be only 1 value
+    List<Double> actuals = read(columnBuilder.getColumnQualifier("value"), Double.class);
+    Assert.assertEquals(1, actuals.size());
+
     // verify - there are 5 'HTTP' messages each with a length of 20, thus the average should be 20
-    double actual = read(columnBuilder.getColumnQualifier("value"), Double.class);
+    double actual = actuals.get(0);
     Assert.assertEquals(20.0, actual, 0.01);
   }
 
@@ -193,13 +212,19 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
     fluxComponent.submitTopology();
     kafkaComponent.writeMessages(Constants.INDEXING_TOPIC, input);
 
+    // expect 3 values written by the profile; one for each host
+    final int expected = 3;
+
     // verify - ensure the profile is being persisted
-    waitOrTimeout(() -> profilerTable.getPutLog().size() > 0,
+    waitOrTimeout(() -> profilerTable.getPutLog().size() >= expected,
             timeout(seconds(90)));
 
-    // verify - the profile literally writes 10 as an integer
-    int actual = read(columnBuilder.getColumnQualifier("value"), Integer.class);
-    Assert.assertEquals(10, actual);
+    // verify - the profile sees messages from 3 hosts; 10.0.0.[1-3]
+    List<Integer> actuals = read(columnBuilder.getColumnQualifier("value"), Integer.class);
+    Assert.assertEquals(3, actuals.size());
+
+    // verify - the profile writes 10 as an integer
+    actuals.forEach(actual -> Assert.assertEquals(10.0, actual, 0.01));
   }
 
   @Test
@@ -215,9 +240,13 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
     waitOrTimeout(() -> profilerTable.getPutLog().size() > 0,
             timeout(seconds(90)));
 
+    List<Double> actuals = read(columnBuilder.getColumnQualifier("value"), Double.class);
+
+    // verify - the profile only cares about HTTP and only 1 host sends HTTP
+    Assert.assertEquals(1, actuals.size());
+
     // verify - the 70th percentile of 5 x 20s = 20.0
-    double actual = read(columnBuilder.getColumnQualifier("value"), Double.class);
-    Assert.assertEquals(20.0, actual, 0.01);
+    actuals.forEach(actual -> Assert.assertEquals(20.0, actual, 0.01));
   }
 
   /**
@@ -228,18 +257,19 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
    * @param <T> The expected type of the result.
    * @return The value contained within the column.
    */
-  private <T> T read(byte[] column, Class<T> clazz) throws IOException {
+  private <T> List<T> read(byte[] column, Class<T> clazz) throws IOException {
+    List<T> results = new ArrayList<>();
+
     final byte[] cf = Bytes.toBytes(columnFamily);
     ResultScanner scanner = profilerTable.getScanner(cf, column);
-
-    for (Result result : scanner) {
-      if(result.containsColumn(cf, column)) {
+    for (Result result : scanner.next(10)) {
+      if (result.containsColumn(cf, column)) {
         byte[] raw = result.getValue(cf, column);
-        return SerDeUtils.fromBytes(raw, clazz);
+        results.add(SerDeUtils.fromBytes(raw, clazz));
       }
     }
 
-    throw new IllegalStateException("No results found");
+    return results;
   }
 
   public void setup(String pathToConfig) throws Exception {
@@ -260,11 +290,14 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
       setProperty("profiler.input.topic", Constants.INDEXING_TOPIC);
       setProperty("profiler.period.duration", "5");
       setProperty("profiler.period.duration.units", "SECONDS");
+      setProperty("profiler.ttl", "30");
+      setProperty("profiler.ttl.units", "MINUTES");
       setProperty("profiler.hbase.salt.divisor", "10");
       setProperty("profiler.hbase.table", tableName);
       setProperty("profiler.hbase.column.family", columnFamily);
       setProperty("profiler.hbase.batch", "10");
       setProperty("profiler.hbase.flush.interval.seconds", "1");
+      setProperty("profiler.profile.ttl", "20");
       setProperty("hbase.provider.impl", "" + MockTableProvider.class.getName());
     }};
 
@@ -305,6 +338,7 @@ public class ProfilerIntegrationTest extends BaseIntegrationTest {
 
   @After
   public void tearDown() throws Exception {
+    MockHTable.Provider.clear();
     if (runner != null) {
       runner.stop();
     }