You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2017/07/03 14:41:48 UTC
[30/40] storm git commit: STORM-2489: Overlap and data loss on
WindowedBolt based on Duration
STORM-2489: Overlap and data loss on WindowedBolt based on Duration
- Fixed time eviction to not evict events beyond current timestamp.
- Introduce a processing time delta for dynamically adjusting window boundaries
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ce8081e9
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ce8081e9
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ce8081e9
Branch: refs/heads/1.1.x-branch
Commit: ce8081e9d1ebbac18d3de6b3802f21f2b0ea127c
Parents: a8fab3c
Author: Arun Mahadevan <ar...@apache.org>
Authored: Tue Apr 25 12:37:43 2017 +0530
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Thu Jun 29 16:55:16 2017 +0900
----------------------------------------------------------------------
.../storm/topology/WindowedBoltExecutor.java | 2 +-
.../storm/windowing/DefaultEvictionContext.java | 11 ++++++++
.../apache/storm/windowing/EvictionContext.java | 8 ++++++
.../storm/windowing/TimeEvictionPolicy.java | 27 +++++++++++++++++++-
.../storm/windowing/TimeTriggerPolicy.java | 4 ++-
5 files changed, 49 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ce8081e9/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
index fd98274..965d11c 100644
--- a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
+++ b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
@@ -275,7 +275,7 @@ public class WindowedBoltExecutor implements IRichBolt {
this.listener = newWindowLifecycleListener();
this.windowManager = initWindowManager(listener, stormConf, context);
start();
- LOG.debug("Initialized window manager {} ", this.windowManager);
+ LOG.info("Initialized window manager {} ", windowManager);
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/ce8081e9/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
index ef65f66..cc16664 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
@@ -21,6 +21,7 @@ public class DefaultEvictionContext implements EvictionContext {
private final Long referenceTime;
private final Long currentCount;
private final Long slidingCount;
+ private final Long slidingInterval;
public DefaultEvictionContext(Long referenceTime) {
this(referenceTime, null);
@@ -31,9 +32,14 @@ public class DefaultEvictionContext implements EvictionContext {
}
public DefaultEvictionContext(Long referenceTime, Long currentCount, Long slidingCount) {
+ this(referenceTime, currentCount, slidingCount, null);
+ }
+
+ public DefaultEvictionContext(Long referenceTime, Long currentCount, Long slidingCount, Long slidingInterval) {
this.referenceTime = referenceTime;
this.currentCount = currentCount;
this.slidingCount = slidingCount;
+ this.slidingInterval = slidingInterval;
}
@Override
@@ -50,4 +56,9 @@ public class DefaultEvictionContext implements EvictionContext {
public Long getSlidingCount() {
return slidingCount;
}
+
+ @Override
+ public Long getSlidingInterval() {
+ return slidingInterval;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/ce8081e9/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
index 37dcfd9..c5a578a 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
@@ -37,6 +37,14 @@ public interface EvictionContext {
*/
Long getSlidingCount();
+
+ /**
+ * Returns the sliding interval for time based windows
+ *
+ * @return the sliding interval
+ */
+ Long getSlidingInterval();
+
/**
* Returns the current count of events in the queue up to the reference tim
* based on which count based evictions can be performed.
http://git-wip-us.apache.org/repos/asf/storm/blob/ce8081e9/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
index e646207..0ab28a1 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
@@ -17,16 +17,23 @@
*/
package org.apache.storm.windowing;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Eviction policy that evicts events based on time duration.
*/
public class TimeEvictionPolicy<T> implements EvictionPolicy<T> {
+ private static final Logger LOG = LoggerFactory.getLogger(TimeEvictionPolicy.class);
+
private final int windowLength;
/**
* The reference time in millis for window calculations and
* expiring events. If not set it will default to System.currentTimeMillis()
*/
protected Long referenceTime;
+ protected EvictionContext evictionContext;
+ private long delta;
/**
* Constructs a TimeEvictionPolicy that evicts events older
@@ -45,8 +52,10 @@ public class TimeEvictionPolicy<T> implements EvictionPolicy<T> {
public Action evict(Event<T> event) {
long now = referenceTime == null ? System.currentTimeMillis() : referenceTime;
long diff = now - event.getTimestamp();
- if (diff >= windowLength) {
+ if (diff >= (windowLength + delta)) {
return Action.EXPIRE;
+ } else if (diff < 0) { // do not process events beyond current ts
+ return Action.KEEP;
}
return Action.PROCESS;
}
@@ -59,6 +68,22 @@ public class TimeEvictionPolicy<T> implements EvictionPolicy<T> {
@Override
public void setContext(EvictionContext context) {
referenceTime = context.getReferenceTime();
+ EvictionContext prevContext = evictionContext;
+ evictionContext = context;
+ // compute window length adjustment (delta) to account for time drift
+ if (context.getSlidingInterval() != null) {
+ if (prevContext == null) {
+ delta = Integer.MAX_VALUE; // consider all events for the initial window
+ } else {
+ delta = context.getReferenceTime() - prevContext.getReferenceTime() - context.getSlidingInterval();
+ if (Math.abs(delta) > 100) {
+ LOG.warn("Possible clock drift or long running computation in window; " +
+ "Previous eviction time: {}, current eviction time: {}",
+ prevContext.getReferenceTime(),
+ context.getReferenceTime());
+ }
+ }
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/ce8081e9/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
index b057afb..f6e0197 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
@@ -109,13 +109,15 @@ public class TimeTriggerPolicy<T> implements TriggerPolicy<T> {
return new Runnable() {
@Override
public void run() {
+ // do not process current timestamp since tuples might arrive while the trigger is executing
+ long now = System.currentTimeMillis() - 1;
try {
/*
* set the current timestamp as the reference time for the eviction policy
* to evict the events
*/
if (evictionPolicy != null) {
- evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis()));
+ evictionPolicy.setContext(new DefaultEvictionContext(now, null, null, duration));
}
handler.onTrigger();
} catch (Throwable th) {