You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2017/07/03 14:41:48 UTC

[30/40] storm git commit: STORM-2489: Overlap and data loss on WindowedBolt based on Duration

STORM-2489: Overlap and data loss on WindowedBolt based on Duration

- Fixed time eviction to not evict events beyond current timestamp.
- Introduce a processing time delta for dynamically adjusting window boundaries


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ce8081e9
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ce8081e9
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ce8081e9

Branch: refs/heads/1.1.x-branch
Commit: ce8081e9d1ebbac18d3de6b3802f21f2b0ea127c
Parents: a8fab3c
Author: Arun Mahadevan <ar...@apache.org>
Authored: Tue Apr 25 12:37:43 2017 +0530
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Thu Jun 29 16:55:16 2017 +0900

----------------------------------------------------------------------
 .../storm/topology/WindowedBoltExecutor.java    |  2 +-
 .../storm/windowing/DefaultEvictionContext.java | 11 ++++++++
 .../apache/storm/windowing/EvictionContext.java |  8 ++++++
 .../storm/windowing/TimeEvictionPolicy.java     | 27 +++++++++++++++++++-
 .../storm/windowing/TimeTriggerPolicy.java      |  4 ++-
 5 files changed, 49 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/ce8081e9/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
index fd98274..965d11c 100644
--- a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
+++ b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
@@ -275,7 +275,7 @@ public class WindowedBoltExecutor implements IRichBolt {
         this.listener = newWindowLifecycleListener();
         this.windowManager = initWindowManager(listener, stormConf, context);
         start();
-        LOG.debug("Initialized window manager {} ", this.windowManager);
+        LOG.info("Initialized window manager {} ", windowManager);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/storm/blob/ce8081e9/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
index ef65f66..cc16664 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
@@ -21,6 +21,7 @@ public class DefaultEvictionContext implements EvictionContext {
     private final Long referenceTime;
     private final Long currentCount;
     private final Long slidingCount;
+    private final Long slidingInterval;
 
     public DefaultEvictionContext(Long referenceTime) {
         this(referenceTime, null);
@@ -31,9 +32,14 @@ public class DefaultEvictionContext implements EvictionContext {
     }
 
     public DefaultEvictionContext(Long referenceTime, Long currentCount, Long slidingCount) {
+        this(referenceTime, currentCount, slidingCount, null);
+    }
+
+    public DefaultEvictionContext(Long referenceTime, Long currentCount, Long slidingCount, Long slidingInterval) {
         this.referenceTime = referenceTime;
         this.currentCount = currentCount;
         this.slidingCount = slidingCount;
+        this.slidingInterval = slidingInterval;
     }
 
     @Override
@@ -50,4 +56,9 @@ public class DefaultEvictionContext implements EvictionContext {
     public Long getSlidingCount() {
         return slidingCount;
     }
+
+    @Override
+    public Long getSlidingInterval() {
+        return slidingInterval;
+    }
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/ce8081e9/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
index 37dcfd9..c5a578a 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
@@ -37,6 +37,14 @@ public interface EvictionContext {
      */
     Long getSlidingCount();
 
+
+    /**
+     * Returns the sliding interval for time based windows
+     *
+     * @return the sliding interval
+     */
+    Long getSlidingInterval();
+
     /**
      * Returns the current count of events in the queue up to the reference tim
      * based on which count based evictions can be performed.

http://git-wip-us.apache.org/repos/asf/storm/blob/ce8081e9/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
index e646207..0ab28a1 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
@@ -17,16 +17,23 @@
  */
 package org.apache.storm.windowing;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * Eviction policy that evicts events based on time duration.
  */
 public class TimeEvictionPolicy<T> implements EvictionPolicy<T> {
+    private static final Logger LOG = LoggerFactory.getLogger(TimeEvictionPolicy.class);
+
     private final int windowLength;
     /**
      * The reference time in millis for window calculations and
      * expiring events. If not set it will default to System.currentTimeMillis()
      */
     protected Long referenceTime;
+    protected EvictionContext evictionContext;
+    private long delta;
 
     /**
      * Constructs a TimeEvictionPolicy that evicts events older
@@ -45,8 +52,10 @@ public class TimeEvictionPolicy<T> implements EvictionPolicy<T> {
     public Action evict(Event<T> event) {
         long now = referenceTime == null ? System.currentTimeMillis() : referenceTime;
         long diff = now - event.getTimestamp();
-        if (diff >= windowLength) {
+        if (diff >= (windowLength + delta)) {
             return Action.EXPIRE;
+        } else if (diff < 0) { // do not process events beyond current ts
+            return Action.KEEP;
         }
         return Action.PROCESS;
     }
@@ -59,6 +68,22 @@ public class TimeEvictionPolicy<T> implements EvictionPolicy<T> {
     @Override
     public void setContext(EvictionContext context) {
         referenceTime = context.getReferenceTime();
+        EvictionContext prevContext = evictionContext;
+        evictionContext = context;
+        // compute window length adjustment (delta) to account for time drift
+        if (context.getSlidingInterval() != null) {
+            if (prevContext == null) {
+                delta = Integer.MAX_VALUE; // consider all events for the initial window
+            } else {
+                delta = context.getReferenceTime() - prevContext.getReferenceTime() - context.getSlidingInterval();
+                if (Math.abs(delta) > 100) {
+                    LOG.warn("Possible clock drift or long running computation in window; " +
+                                    "Previous eviction time: {}, current eviction time: {}",
+                            prevContext.getReferenceTime(),
+                            context.getReferenceTime());
+                }
+            }
+        }
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/storm/blob/ce8081e9/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
index b057afb..f6e0197 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
@@ -109,13 +109,15 @@ public class TimeTriggerPolicy<T> implements TriggerPolicy<T> {
         return new Runnable() {
             @Override
             public void run() {
+                // do not process current timestamp since tuples might arrive while the trigger is executing
+                long now = System.currentTimeMillis() - 1;
                 try {
                     /*
                      * set the current timestamp as the reference time for the eviction policy
                      * to evict the events
                      */
                     if (evictionPolicy != null) {
-                        evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis()));
+                        evictionPolicy.setContext(new DefaultEvictionContext(now, null, null, duration));
                     }
                     handler.onTrigger();
                 } catch (Throwable th) {