You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2016/07/21 17:27:27 UTC

[01/13] storm git commit: [STORM-1964] Unexpected behavior when using count window together with timestamp extraction

Repository: storm
Updated Branches:
  refs/heads/1.0.x-branch 7936de9f1 -> bb1ea75e2
  refs/heads/1.x-branch 88e8f17d4 -> 6deb60fb4
  refs/heads/master e5be9863c -> 5e5bb8f1f


[STORM-1964] Unexpected behavior when using count window together with timestamp extraction

Do not use timestamp to determine the event count in WatermarkCountEvictionPolicy
when count based trigger is used.


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3cbd26ec
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3cbd26ec
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3cbd26ec

Branch: refs/heads/master
Commit: 3cbd26ec275bb052195a038b49853ed1cfc84806
Parents: 05e3289
Author: Arun Mahadevan <ar...@apache.org>
Authored: Thu Jul 14 00:00:57 2016 +0530
Committer: Arun Mahadevan <ar...@apache.org>
Committed: Fri Jul 15 11:59:48 2016 +0530

----------------------------------------------------------------------
 .../storm/topology/WindowedBoltExecutor.java    |  2 +-
 .../storm/windowing/CountEvictionPolicy.java    | 12 ++--
 .../storm/windowing/CountTriggerPolicy.java     |  2 +-
 .../storm/windowing/DefaultEvictionContext.java | 53 +++++++++++++++
 .../apache/storm/windowing/EvictionContext.java | 47 +++++++++++++
 .../apache/storm/windowing/EvictionPolicy.java  |  3 +-
 .../storm/windowing/TimeEvictionPolicy.java     |  4 +-
 .../storm/windowing/TimeTriggerPolicy.java      |  2 +-
 .../windowing/WatermarkCountEvictionPolicy.java | 27 +++++---
 .../windowing/WatermarkCountTriggerPolicy.java  |  2 +-
 .../windowing/WatermarkTimeTriggerPolicy.java   |  3 +-
 .../storm/windowing/WindowManagerTest.java      | 69 ++++++++++++++++++--
 12 files changed, 197 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
index fe0f4da..3f245e1 100644
--- a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
+++ b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
@@ -253,7 +253,7 @@ public class WindowedBoltExecutor implements IRichBolt {
                                                     WindowManager<Tuple> manager) {
         if (windowLengthCount != null) {
             if (isTupleTs()) {
-                return new WatermarkCountEvictionPolicy<>(windowLengthCount.value, manager);
+                return new WatermarkCountEvictionPolicy<>(windowLengthCount.value);
             } else {
                 return new CountEvictionPolicy<>(windowLengthCount.value);
             }

http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
index 0e7b233..fb12202 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
@@ -17,7 +17,7 @@
  */
 package org.apache.storm.windowing;
 
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * An eviction policy that tracks event counts and can
@@ -26,12 +26,12 @@ import java.util.concurrent.atomic.AtomicInteger;
  * @param <T> the type of event tracked by this policy.
  */
 public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
-    private final int threshold;
-    protected final AtomicInteger currentCount;
+    protected final int threshold;
+    protected final AtomicLong currentCount;
 
     public CountEvictionPolicy(int count) {
         this.threshold = count;
-        this.currentCount = new AtomicInteger();
+        this.currentCount = new AtomicLong();
     }
 
     @Override
@@ -41,7 +41,7 @@ public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
          * return if the event should be evicted
          */
         while (true) {
-            int curVal = currentCount.get();
+            long curVal = currentCount.get();
             if (curVal > threshold) {
                 if (currentCount.compareAndSet(curVal, curVal - 1)) {
                     return Action.EXPIRE;
@@ -61,7 +61,7 @@ public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
     }
 
     @Override
-    public void setContext(Object context) {
+    public void setContext(EvictionContext context) {
         // NOOP
     }
 

http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
index 2594c67..17750b6 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
@@ -44,7 +44,7 @@ public class CountTriggerPolicy<T> implements TriggerPolicy<T> {
     public void track(Event<T> event) {
         if (started && !event.isWatermark()) {
             if (currentCount.incrementAndGet() >= count) {
-                evictionPolicy.setContext(System.currentTimeMillis());
+                evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis()));
                 handler.onTrigger();
             }
         }

http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
new file mode 100644
index 0000000..ef65f66
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.windowing;
+
+public class DefaultEvictionContext implements EvictionContext {
+    private final Long referenceTime;
+    private final Long currentCount;
+    private final Long slidingCount;
+
+    public DefaultEvictionContext(Long referenceTime) {
+        this(referenceTime, null);
+    }
+
+    public DefaultEvictionContext(Long referenceTime, Long currentCount) {
+        this(referenceTime, currentCount, null);
+    }
+
+    public DefaultEvictionContext(Long referenceTime, Long currentCount, Long slidingCount) {
+        this.referenceTime = referenceTime;
+        this.currentCount = currentCount;
+        this.slidingCount = slidingCount;
+    }
+
+    @Override
+    public Long getReferenceTime() {
+        return referenceTime;
+    }
+
+    @Override
+    public Long getCurrentCount() {
+        return currentCount;
+    }
+
+    @Override
+    public Long getSlidingCount() {
+        return slidingCount;
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
new file mode 100644
index 0000000..37dcfd9
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.windowing;
+
+/**
+ * Context information that can be used by the eviction policy
+ */
+public interface EvictionContext {
+    /**
+     * Returns the reference time that the eviction policy could use to
+     * evict the events. In the case of event time processing, this would be
+     * the watermark time.
+     *
+     * @return the reference time in millis
+     */
+    Long getReferenceTime();
+
+    /**
+     * Returns the sliding count for count based windows
+     *
+     * @return the sliding count
+     */
+    Long getSlidingCount();
+
+    /**
+     * Returns the current count of events in the queue up to the reference tim
+     * based on which count based evictions can be performed.
+     *
+     * @return the current count
+     */
+    Long getCurrentCount();
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
index 8128d80..05e4d93 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
@@ -70,5 +70,6 @@ public interface EvictionPolicy<T> {
      *
      * @param context
      */
-    void setContext(Object context);
+    void setContext(EvictionContext context);
+
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
index a2db239..e646207 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
@@ -57,8 +57,8 @@ public class TimeEvictionPolicy<T> implements EvictionPolicy<T> {
     }
 
     @Override
-    public void setContext(Object context) {
-        referenceTime = ((Number) context).longValue();
+    public void setContext(EvictionContext context) {
+        referenceTime = context.getReferenceTime();
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
index 1ad7714..b057afb 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
@@ -115,7 +115,7 @@ public class TimeTriggerPolicy<T> implements TriggerPolicy<T> {
                      * to evict the events
                      */
                     if (evictionPolicy != null) {
-                        evictionPolicy.setContext(System.currentTimeMillis());
+                        evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis()));
                     }
                     handler.onTrigger();
                 } catch (Throwable th) {

http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
index e1af6fb..74240bb 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
@@ -24,25 +24,29 @@ package org.apache.storm.windowing;
  * @param <T> the type of event tracked by this policy.
  */
 public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
-    private final WindowManager<T> windowManager;
     /*
      * The reference time in millis for window calculations and
      * expiring events. If not set it will default to System.currentTimeMillis()
      */
     private long referenceTime;
+    private long processed = 0L;
 
-    public WatermarkCountEvictionPolicy(int count, WindowManager<T> windowManager) {
+    public WatermarkCountEvictionPolicy(int count) {
         super(count);
-        this.windowManager = windowManager;
     }
 
     @Override
     public Action evict(Event<T> event) {
-        if (event.getTimestamp() <= referenceTime) {
-            return super.evict(event);
+        Action action;
+        if (event.getTimestamp() <= referenceTime && processed < currentCount.get()) {
+            action = super.evict(event);
+            if (action == Action.PROCESS) {
+                ++processed;
+            }
         } else {
-            return Action.KEEP;
+            action = Action.KEEP;
         }
+        return action;
     }
 
     @Override
@@ -51,9 +55,14 @@ public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
     }
 
     @Override
-    public void setContext(Object context) {
-        referenceTime = (Long) context;
-        currentCount.set(windowManager.getEventCount(referenceTime));
+    public void setContext(EvictionContext context) {
+        referenceTime = context.getReferenceTime();
+        if (context.getCurrentCount() != null) {
+            currentCount.set(context.getCurrentCount());
+        } else {
+            currentCount.set(processed + context.getSlidingCount());
+        }
+        processed = 0;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
index 391efee..3cfcaad 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
@@ -74,7 +74,7 @@ public class WatermarkCountTriggerPolicy<T> implements TriggerPolicy<T> {
         long watermarkTs = waterMarkEvent.getTimestamp();
         List<Long> eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, count);
         for (long ts : eventTs) {
-            evictionPolicy.setContext(ts);
+            evictionPolicy.setContext(new DefaultEvictionContext(ts, null, Long.valueOf(count)));
             handler.onTrigger();
             lastProcessedTs = ts;
         }

http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
index 3734c93..00620c4 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
@@ -75,7 +75,8 @@ public class WatermarkTimeTriggerPolicy<T> implements TriggerPolicy<T> {
         long windowEndTs = nextWindowEndTs;
         LOG.debug("Window end ts {} Watermark ts {}", windowEndTs, watermarkTs);
         while (windowEndTs <= watermarkTs) {
-            evictionPolicy.setContext(windowEndTs);
+            long currentCount = windowManager.getEventCount(windowEndTs);
+            evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount));
             if (handler.onTrigger()) {
                 windowEndTs += slidingIntervalMs;
             } else {

http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java b/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
index a545427..6645566 100644
--- a/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
@@ -187,7 +187,7 @@ public class WindowManagerTest {
             windowManager.add(i, now - 1000);
         }
         // simulate the time trigger by setting the reference time and invoking onTrigger() manually
-        evictionPolicy.setContext(now + 100);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 100));
         windowManager.onTrigger();
 
         // 100 events with past ts should expire
@@ -210,7 +210,7 @@ public class WindowManagerTest {
         }
         activationsEvents.addAll(newEvents);
         // simulate the time trigger by setting the reference time and invoking onTrigger() manually
-        evictionPolicy.setContext(now + 200);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 200));
         windowManager.onTrigger();
         assertTrue(listener.onExpiryEvents.isEmpty());
         assertEquals(activationsEvents, listener.onActivationEvents);
@@ -236,20 +236,20 @@ public class WindowManagerTest {
             windowManager.add(i);
         }
         // simulate the time trigger by setting the reference time and invoking onTrigger() manually
-        evictionPolicy.setContext(now + 60);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 60));
         windowManager.onTrigger();
 
         assertEquals(seq(1, 10), listener.onActivationEvents);
         assertTrue(listener.onActivationExpiredEvents.isEmpty());
         listener.clear();
         // wait so all events expire
-        evictionPolicy.setContext(now + 120);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 120));
         windowManager.onTrigger();
 
         assertEquals(seq(1, 10), listener.onExpiryEvents);
         assertTrue(listener.onActivationEvents.isEmpty());
         listener.clear();
-        evictionPolicy.setContext(now + 180);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 180));
         windowManager.onTrigger();
         assertTrue(listener.onActivationExpiredEvents.isEmpty());
         assertTrue(listener.onActivationEvents.isEmpty());
@@ -354,7 +354,7 @@ public class WindowManagerTest {
 
     @Test
     public void testCountBasedWindowWithEventTs() throws Exception {
-        EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(3, windowManager);
+        EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(3);
         windowManager.setEvictionPolicy(evictionPolicy);
         TriggerPolicy<Integer> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager);
         triggerPolicy.start();
@@ -430,7 +430,64 @@ public class WindowManagerTest {
         assertEquals(seq(9), listener.allOnActivationEvents.get(0));
         assertEquals(seq(9, 12), listener.allOnActivationEvents.get(1));
     }
+
+    @Test
+    public void testCountBasedTumblingWithSameEventTs() throws Exception {
+        EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(2);
+        windowManager.setEvictionPolicy(evictionPolicy);
+        TriggerPolicy<Integer> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager);
+        triggerPolicy.start();
+        windowManager.setTriggerPolicy(triggerPolicy);
+
+        windowManager.add(1, 10);
+        windowManager.add(2, 10);
+        windowManager.add(3, 11);
+        windowManager.add(4, 12);
+        windowManager.add(5, 12);
+        windowManager.add(6, 12);
+        windowManager.add(7, 12);
+        windowManager.add(8, 13);
+        windowManager.add(9, 14);
+        windowManager.add(10, 15);
+
+        windowManager.add(new WaterMarkEvent<Integer>(20));
+        assertEquals(5, listener.allOnActivationEvents.size());
+        assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0));
+        assertEquals(seq(3, 4), listener.allOnActivationEvents.get(1));
+        assertEquals(seq(5, 6), listener.allOnActivationEvents.get(2));
+        assertEquals(seq(7, 8), listener.allOnActivationEvents.get(3));
+        assertEquals(seq(9, 10), listener.allOnActivationEvents.get(4));
+    }
+
     @Test
+    public void testCountBasedSlidingWithSameEventTs() throws Exception {
+        EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(5);
+        windowManager.setEvictionPolicy(evictionPolicy);
+        TriggerPolicy<Integer> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager);
+        triggerPolicy.start();
+        windowManager.setTriggerPolicy(triggerPolicy);
+
+        windowManager.add(1, 10);
+        windowManager.add(2, 10);
+        windowManager.add(3, 11);
+        windowManager.add(4, 12);
+        windowManager.add(5, 12);
+        windowManager.add(6, 12);
+        windowManager.add(7, 12);
+        windowManager.add(8, 13);
+        windowManager.add(9, 14);
+        windowManager.add(10, 15);
+
+        windowManager.add(new WaterMarkEvent<Integer>(20));
+        assertEquals(5, listener.allOnActivationEvents.size());
+        assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0));
+        assertEquals(seq(1, 4), listener.allOnActivationEvents.get(1));
+        assertEquals(seq(2, 6), listener.allOnActivationEvents.get(2));
+        assertEquals(seq(4, 8), listener.allOnActivationEvents.get(3));
+        assertEquals(seq(6, 10), listener.allOnActivationEvents.get(4));
+
+    }
+        @Test
     public void testEventTimeLag() throws Exception {
         EvictionPolicy<Integer> evictionPolicy = new WatermarkTimeEvictionPolicy<>(20, 5);
         windowManager.setEvictionPolicy(evictionPolicy);


[10/13] storm git commit: Unexpected behavior when using count window together with timestamp extraction

Posted by pt...@apache.org.
Unexpected behavior when using count window together with timestamp extraction

Do not use timestamp to determine the event count in WatermarkCountEvictionPolicy
when count based trigger is used.

Signed-off-by: P. Taylor Goetz <pt...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f1eacf1c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f1eacf1c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f1eacf1c

Branch: refs/heads/1.x-branch
Commit: f1eacf1cef3af63ab8071026145cb6bbc6fed2c6
Parents: 88e8f17
Author: Arun Mahadevan <ar...@apache.org>
Authored: Thu Jul 14 00:00:57 2016 +0530
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 13:24:35 2016 -0400

----------------------------------------------------------------------
 .../storm/topology/WindowedBoltExecutor.java    |  2 +-
 .../storm/windowing/CountEvictionPolicy.java    | 12 ++--
 .../storm/windowing/CountTriggerPolicy.java     |  2 +-
 .../storm/windowing/DefaultEvictionContext.java | 53 +++++++++++++++
 .../apache/storm/windowing/EvictionContext.java | 47 +++++++++++++
 .../apache/storm/windowing/EvictionPolicy.java  |  3 +-
 .../storm/windowing/TimeEvictionPolicy.java     |  4 +-
 .../storm/windowing/TimeTriggerPolicy.java      |  2 +-
 .../windowing/WatermarkCountEvictionPolicy.java | 27 +++++---
 .../windowing/WatermarkCountTriggerPolicy.java  |  2 +-
 .../windowing/WatermarkTimeTriggerPolicy.java   |  3 +-
 .../storm/windowing/WindowManagerTest.java      | 69 ++++++++++++++++++--
 12 files changed, 197 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
index fe0f4da..3f245e1 100644
--- a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
+++ b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
@@ -253,7 +253,7 @@ public class WindowedBoltExecutor implements IRichBolt {
                                                     WindowManager<Tuple> manager) {
         if (windowLengthCount != null) {
             if (isTupleTs()) {
-                return new WatermarkCountEvictionPolicy<>(windowLengthCount.value, manager);
+                return new WatermarkCountEvictionPolicy<>(windowLengthCount.value);
             } else {
                 return new CountEvictionPolicy<>(windowLengthCount.value);
             }

http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
index 0e7b233..fb12202 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
@@ -17,7 +17,7 @@
  */
 package org.apache.storm.windowing;
 
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * An eviction policy that tracks event counts and can
@@ -26,12 +26,12 @@ import java.util.concurrent.atomic.AtomicInteger;
  * @param <T> the type of event tracked by this policy.
  */
 public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
-    private final int threshold;
-    protected final AtomicInteger currentCount;
+    protected final int threshold;
+    protected final AtomicLong currentCount;
 
     public CountEvictionPolicy(int count) {
         this.threshold = count;
-        this.currentCount = new AtomicInteger();
+        this.currentCount = new AtomicLong();
     }
 
     @Override
@@ -41,7 +41,7 @@ public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
          * return if the event should be evicted
          */
         while (true) {
-            int curVal = currentCount.get();
+            long curVal = currentCount.get();
             if (curVal > threshold) {
                 if (currentCount.compareAndSet(curVal, curVal - 1)) {
                     return Action.EXPIRE;
@@ -61,7 +61,7 @@ public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
     }
 
     @Override
-    public void setContext(Object context) {
+    public void setContext(EvictionContext context) {
         // NOOP
     }
 

http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
index 2594c67..17750b6 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
@@ -44,7 +44,7 @@ public class CountTriggerPolicy<T> implements TriggerPolicy<T> {
     public void track(Event<T> event) {
         if (started && !event.isWatermark()) {
             if (currentCount.incrementAndGet() >= count) {
-                evictionPolicy.setContext(System.currentTimeMillis());
+                evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis()));
                 handler.onTrigger();
             }
         }

http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
new file mode 100644
index 0000000..ef65f66
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.windowing;
+
+public class DefaultEvictionContext implements EvictionContext {
+    private final Long referenceTime;
+    private final Long currentCount;
+    private final Long slidingCount;
+
+    public DefaultEvictionContext(Long referenceTime) {
+        this(referenceTime, null);
+    }
+
+    public DefaultEvictionContext(Long referenceTime, Long currentCount) {
+        this(referenceTime, currentCount, null);
+    }
+
+    public DefaultEvictionContext(Long referenceTime, Long currentCount, Long slidingCount) {
+        this.referenceTime = referenceTime;
+        this.currentCount = currentCount;
+        this.slidingCount = slidingCount;
+    }
+
+    @Override
+    public Long getReferenceTime() {
+        return referenceTime;
+    }
+
+    @Override
+    public Long getCurrentCount() {
+        return currentCount;
+    }
+
+    @Override
+    public Long getSlidingCount() {
+        return slidingCount;
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
new file mode 100644
index 0000000..37dcfd9
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.windowing;
+
+/**
+ * Context information that can be used by the eviction policy
+ */
+public interface EvictionContext {
+    /**
+     * Returns the reference time that the eviction policy could use to
+     * evict the events. In the case of event time processing, this would be
+     * the watermark time.
+     *
+     * @return the reference time in millis
+     */
+    Long getReferenceTime();
+
+    /**
+     * Returns the sliding count for count based windows
+     *
+     * @return the sliding count
+     */
+    Long getSlidingCount();
+
+    /**
+     * Returns the current count of events in the queue up to the reference tim
+     * based on which count based evictions can be performed.
+     *
+     * @return the current count
+     */
+    Long getCurrentCount();
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
index 8128d80..05e4d93 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
@@ -70,5 +70,6 @@ public interface EvictionPolicy<T> {
      *
      * @param context
      */
-    void setContext(Object context);
+    void setContext(EvictionContext context);
+
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
index a2db239..e646207 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
@@ -57,8 +57,8 @@ public class TimeEvictionPolicy<T> implements EvictionPolicy<T> {
     }
 
     @Override
-    public void setContext(Object context) {
-        referenceTime = ((Number) context).longValue();
+    public void setContext(EvictionContext context) {
+        referenceTime = context.getReferenceTime();
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
index 1ad7714..b057afb 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
@@ -115,7 +115,7 @@ public class TimeTriggerPolicy<T> implements TriggerPolicy<T> {
                      * to evict the events
                      */
                     if (evictionPolicy != null) {
-                        evictionPolicy.setContext(System.currentTimeMillis());
+                        evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis()));
                     }
                     handler.onTrigger();
                 } catch (Throwable th) {

http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
index e1af6fb..74240bb 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
@@ -24,25 +24,29 @@ package org.apache.storm.windowing;
  * @param <T> the type of event tracked by this policy.
  */
 public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
-    private final WindowManager<T> windowManager;
     /*
      * The reference time in millis for window calculations and
      * expiring events. If not set it will default to System.currentTimeMillis()
      */
     private long referenceTime;
+    private long processed = 0L;
 
-    public WatermarkCountEvictionPolicy(int count, WindowManager<T> windowManager) {
+    public WatermarkCountEvictionPolicy(int count) {
         super(count);
-        this.windowManager = windowManager;
     }
 
     @Override
     public Action evict(Event<T> event) {
-        if (event.getTimestamp() <= referenceTime) {
-            return super.evict(event);
+        Action action;
+        if (event.getTimestamp() <= referenceTime && processed < currentCount.get()) {
+            action = super.evict(event);
+            if (action == Action.PROCESS) {
+                ++processed;
+            }
         } else {
-            return Action.KEEP;
+            action = Action.KEEP;
         }
+        return action;
     }
 
     @Override
@@ -51,9 +55,14 @@ public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
     }
 
     @Override
-    public void setContext(Object context) {
-        referenceTime = (Long) context;
-        currentCount.set(windowManager.getEventCount(referenceTime));
+    public void setContext(EvictionContext context) {
+        referenceTime = context.getReferenceTime();
+        if (context.getCurrentCount() != null) {
+            currentCount.set(context.getCurrentCount());
+        } else {
+            currentCount.set(processed + context.getSlidingCount());
+        }
+        processed = 0;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
index 391efee..3cfcaad 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
@@ -74,7 +74,7 @@ public class WatermarkCountTriggerPolicy<T> implements TriggerPolicy<T> {
         long watermarkTs = waterMarkEvent.getTimestamp();
         List<Long> eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, count);
         for (long ts : eventTs) {
-            evictionPolicy.setContext(ts);
+            evictionPolicy.setContext(new DefaultEvictionContext(ts, null, Long.valueOf(count)));
             handler.onTrigger();
             lastProcessedTs = ts;
         }

http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
index 3734c93..00620c4 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
@@ -75,7 +75,8 @@ public class WatermarkTimeTriggerPolicy<T> implements TriggerPolicy<T> {
         long windowEndTs = nextWindowEndTs;
         LOG.debug("Window end ts {} Watermark ts {}", windowEndTs, watermarkTs);
         while (windowEndTs <= watermarkTs) {
-            evictionPolicy.setContext(windowEndTs);
+            long currentCount = windowManager.getEventCount(windowEndTs);
+            evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount));
             if (handler.onTrigger()) {
                 windowEndTs += slidingIntervalMs;
             } else {

http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java b/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
index a545427..6645566 100644
--- a/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
@@ -187,7 +187,7 @@ public class WindowManagerTest {
             windowManager.add(i, now - 1000);
         }
         // simulate the time trigger by setting the reference time and invoking onTrigger() manually
-        evictionPolicy.setContext(now + 100);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 100));
         windowManager.onTrigger();
 
         // 100 events with past ts should expire
@@ -210,7 +210,7 @@ public class WindowManagerTest {
         }
         activationsEvents.addAll(newEvents);
         // simulate the time trigger by setting the reference time and invoking onTrigger() manually
-        evictionPolicy.setContext(now + 200);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 200));
         windowManager.onTrigger();
         assertTrue(listener.onExpiryEvents.isEmpty());
         assertEquals(activationsEvents, listener.onActivationEvents);
@@ -236,20 +236,20 @@ public class WindowManagerTest {
             windowManager.add(i);
         }
         // simulate the time trigger by setting the reference time and invoking onTrigger() manually
-        evictionPolicy.setContext(now + 60);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 60));
         windowManager.onTrigger();
 
         assertEquals(seq(1, 10), listener.onActivationEvents);
         assertTrue(listener.onActivationExpiredEvents.isEmpty());
         listener.clear();
         // wait so all events expire
-        evictionPolicy.setContext(now + 120);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 120));
         windowManager.onTrigger();
 
         assertEquals(seq(1, 10), listener.onExpiryEvents);
         assertTrue(listener.onActivationEvents.isEmpty());
         listener.clear();
-        evictionPolicy.setContext(now + 180);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 180));
         windowManager.onTrigger();
         assertTrue(listener.onActivationExpiredEvents.isEmpty());
         assertTrue(listener.onActivationEvents.isEmpty());
@@ -354,7 +354,7 @@ public class WindowManagerTest {
 
     @Test
     public void testCountBasedWindowWithEventTs() throws Exception {
-        EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(3, windowManager);
+        EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(3);
         windowManager.setEvictionPolicy(evictionPolicy);
         TriggerPolicy<Integer> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager);
         triggerPolicy.start();
@@ -430,7 +430,64 @@ public class WindowManagerTest {
         assertEquals(seq(9), listener.allOnActivationEvents.get(0));
         assertEquals(seq(9, 12), listener.allOnActivationEvents.get(1));
     }
+
+    @Test
+    public void testCountBasedTumblingWithSameEventTs() throws Exception {
+        EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(2);
+        windowManager.setEvictionPolicy(evictionPolicy);
+        TriggerPolicy<Integer> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager);
+        triggerPolicy.start();
+        windowManager.setTriggerPolicy(triggerPolicy);
+
+        windowManager.add(1, 10);
+        windowManager.add(2, 10);
+        windowManager.add(3, 11);
+        windowManager.add(4, 12);
+        windowManager.add(5, 12);
+        windowManager.add(6, 12);
+        windowManager.add(7, 12);
+        windowManager.add(8, 13);
+        windowManager.add(9, 14);
+        windowManager.add(10, 15);
+
+        windowManager.add(new WaterMarkEvent<Integer>(20));
+        assertEquals(5, listener.allOnActivationEvents.size());
+        assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0));
+        assertEquals(seq(3, 4), listener.allOnActivationEvents.get(1));
+        assertEquals(seq(5, 6), listener.allOnActivationEvents.get(2));
+        assertEquals(seq(7, 8), listener.allOnActivationEvents.get(3));
+        assertEquals(seq(9, 10), listener.allOnActivationEvents.get(4));
+    }
+
     @Test
+    public void testCountBasedSlidingWithSameEventTs() throws Exception {
+        EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(5);
+        windowManager.setEvictionPolicy(evictionPolicy);
+        TriggerPolicy<Integer> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager);
+        triggerPolicy.start();
+        windowManager.setTriggerPolicy(triggerPolicy);
+
+        windowManager.add(1, 10);
+        windowManager.add(2, 10);
+        windowManager.add(3, 11);
+        windowManager.add(4, 12);
+        windowManager.add(5, 12);
+        windowManager.add(6, 12);
+        windowManager.add(7, 12);
+        windowManager.add(8, 13);
+        windowManager.add(9, 14);
+        windowManager.add(10, 15);
+
+        windowManager.add(new WaterMarkEvent<Integer>(20));
+        assertEquals(5, listener.allOnActivationEvents.size());
+        assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0));
+        assertEquals(seq(1, 4), listener.allOnActivationEvents.get(1));
+        assertEquals(seq(2, 6), listener.allOnActivationEvents.get(2));
+        assertEquals(seq(4, 8), listener.allOnActivationEvents.get(3));
+        assertEquals(seq(6, 10), listener.allOnActivationEvents.get(4));
+
+    }
+        @Test
     public void testEventTimeLag() throws Exception {
         EvictionPolicy<Integer> evictionPolicy = new WatermarkTimeEvictionPolicy<>(20, 5);
         windowManager.setEvictionPolicy(evictionPolicy);


[02/13] storm git commit: STORM-1890: add maven depency to for ring-core 1.3.1, move org.apache.commons.FileUtils to 1.3.2 as that is needed by ring-core 1.3.1, move compojure to 1.1.9 since it matches 1.3.1

Posted by pt...@apache.org.
STORM-1890: add maven depency to for ring-core 1.3.1, move org.apache.commons.FileUtils to 1.3.2 as that is needed by ring-core 1.3.1, move compojure to 1.1.9 since it matches 1.3.1

Signed-off-by: P. Taylor Goetz <pt...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/51f0daa9
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/51f0daa9
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/51f0daa9

Branch: refs/heads/1.0.x-branch
Commit: 51f0daa9df14f499d83fdd513bbaed1d4f4fcf7b
Parents: 44e8170
Author: Alessandro Bellina <ab...@yahoo-inc.com>
Authored: Thu Jun 16 11:07:48 2016 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 12:14:18 2016 -0400

----------------------------------------------------------------------
 pom.xml            | 11 ++++++++---
 storm-core/pom.xml |  4 ++++
 2 files changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/51f0daa9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9062ada..2ea2184 100644
--- a/pom.xml
+++ b/pom.xml
@@ -200,19 +200,19 @@
         <!-- dependency versions -->
         <clojure.version>1.7.0</clojure.version>
         <java_jmx.version>0.3.1</java_jmx.version>
-        <compojure.version>1.1.3</compojure.version>
+        <compojure.version>1.1.9</compojure.version>
         <hiccup.version>0.3.6</hiccup.version>
         <commons-compress.version>1.4.1</commons-compress.version>
         <commons-io.version>2.5</commons-io.version>
         <commons-lang.version>2.5</commons-lang.version>
         <commons-exec.version>1.1</commons-exec.version>
-        <commons-fileupload.version>1.2.1</commons-fileupload.version>
+        <commons-fileupload.version>1.3.2</commons-fileupload.version>
         <commons-codec.version>1.6</commons-codec.version>
         <commons-cli.version>1.3.1</commons-cli.version>
         <clj-time.version>0.8.0</clj-time.version>
         <curator.version>2.10.0</curator.version>
         <json-simple.version>1.1</json-simple.version>
-        <ring.version>1.3.0</ring.version>
+        <ring.version>1.3.1</ring.version>
         <ring-json.version>0.3.1</ring-json.version>
         <jetty.version>7.6.13.v20130916</jetty.version>
         <clojure.tools.logging.version>0.2.3</clojure.tools.logging.version>
@@ -659,6 +659,11 @@
             </dependency>
             <dependency>
                 <groupId>ring</groupId>
+                <artifactId>ring-core</artifactId>
+                <version>${ring.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>ring</groupId>
                 <artifactId>ring-devel</artifactId>
                 <version>${ring.version}</version>
             </dependency>

http://git-wip-us.apache.org/repos/asf/storm/blob/51f0daa9/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index b79e923..17a9545 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -70,6 +70,10 @@
         </dependency>
         <dependency>
             <groupId>ring</groupId>
+            <artifactId>ring-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ring</groupId>
             <artifactId>ring-devel</artifactId>
         </dependency>
         <dependency>


[11/13] storm git commit: add STORM-1964 to changelog

Posted by pt...@apache.org.
add STORM-1964 to changelog


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6deb60fb
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6deb60fb
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6deb60fb

Branch: refs/heads/1.x-branch
Commit: 6deb60fb488e72b2802e23c7dbf13c53370be116
Parents: f1eacf1
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jul 21 13:25:02 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 13:25:02 2016 -0400

----------------------------------------------------------------------
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/6deb60fb/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 34ac20d..0e8d6d8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
 ## 1.1.0
+ * STORM-1964: Unexpected behavior when using count window together with timestamp extraction
  * STORM-1890: ensure we refetch static resources after package build
  * STORM-1988: Kafka Offset not showing due to bad classpath.
  * STORM-1987: Fix TridentKafkaWordCount arg handling in distributed mode


[03/13] storm git commit: STORM-1890: add temp-file store since wrap-multipart-params for ring 1.3 needs it

Posted by pt...@apache.org.
STORM-1890: add temp-file store since wrap-multipart-params for ring 1.3 needs it

Signed-off-by: P. Taylor Goetz <pt...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b52172b7
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b52172b7
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b52172b7

Branch: refs/heads/1.0.x-branch
Commit: b52172b7b4a8d41510fe96d918fd242b8ba7f4cc
Parents: 28ef601
Author: Alessandro Bellina <ab...@yahoo-inc.com>
Authored: Sat Jun 18 14:04:36 2016 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 12:14:18 2016 -0400

----------------------------------------------------------------------
 storm-core/src/clj/org/apache/storm/ui/core.clj | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/b52172b7/storm-core/src/clj/org/apache/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/ui/core.clj b/storm-core/src/clj/org/apache/storm/ui/core.clj
index 1ab2d33..30c5fc9 100644
--- a/storm-core/src/clj/org/apache/storm/ui/core.clj
+++ b/storm-core/src/clj/org/apache/storm/ui/core.clj
@@ -18,7 +18,8 @@
   (:use compojure.core)
   (:use [clojure.java.shell :only [sh]])
   (:use ring.middleware.reload
-        ring.middleware.multipart-params)
+        ring.middleware.multipart-params
+        ring.middleware.multipart-params.temp-file)
   (:use [ring.middleware.json :only [wrap-json-params]])
   (:use [hiccup core page-helpers])
   (:use [org.apache.storm config util log stats zookeeper converter])


[06/13] storm git commit: add STORM-1890 to changelog

Posted by pt...@apache.org.
add STORM-1890 to changelog


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b068dde1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b068dde1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b068dde1

Branch: refs/heads/1.0.x-branch
Commit: b068dde1e5e21da5f6088398ba01833e8f0411e9
Parents: 51f0daa
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jul 21 12:23:27 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 12:23:27 2016 -0400

----------------------------------------------------------------------
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/b068dde1/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 170aa4e..4182f1a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
 ## 1.0.2
+ * STORM-1890 ensure we refetch static resources after package build
  * STORM-1939: Frequent InterruptedException raised by ShellBoltMessageQueue.poll
  * STORM-1968: Storm logviewer does not work for nimbus.log in secure cluster
  * STORM-1928: ShellSpout should check heartbeat while ShellSpout is waiting for subprocess to sync


[04/13] storm git commit: STORM-1890: add code Cache-Control: no-cache to root html pages (/*.html)

Posted by pt...@apache.org.
STORM-1890: add code Cache-Control: no-cache to root html pages (/*.html)

Signed-off-by: P. Taylor Goetz <pt...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/44e81709
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/44e81709
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/44e81709

Branch: refs/heads/1.0.x-branch
Commit: 44e817092241613f5849ec6e8b6a84a034a2bb0e
Parents: b52172b
Author: Alessandro Bellina <ab...@yahoo-inc.com>
Authored: Sat Jun 18 14:05:27 2016 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 12:14:18 2016 -0400

----------------------------------------------------------------------
 .../src/clj/org/apache/storm/ui/helpers.clj     | 22 ++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/44e81709/storm-core/src/clj/org/apache/storm/ui/helpers.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/ui/helpers.clj b/storm-core/src/clj/org/apache/storm/ui/helpers.clj
index 7ded154..81ceb58 100644
--- a/storm-core/src/clj/org/apache/storm/ui/helpers.clj
+++ b/storm-core/src/clj/org/apache/storm/ui/helpers.clj
@@ -32,18 +32,32 @@
 	   [org.eclipse.jetty.util.ssl SslContextFactory]
            [org.eclipse.jetty.server DispatcherType]
            [org.eclipse.jetty.servlets CrossOriginFilter])
-  (:require [ring.util servlet])
+  (:require [ring.util servlet]
+            [ring.util.response :as response])
   (:require [compojure.route :as route]
             [compojure.handler :as handler])
   (:require [metrics.meters :refer [defmeter mark!]]))
 
 (defmeter num-web-requests)
+
 (defn requests-middleware
-  "Coda Hale metric for counting the number of web requests."
+  "Wrap request with Coda Hale metric for counting the number of web requests,
+  and add Cache-Control: no-cache for html files in root directory (index.html, topology.html, etc)"
   [handler]
   (fn [req]
-    (mark! num-web-requests)
-    (handler req)))
+    (.mark num-web-requests)
+    (let [uri (:uri req)
+          res (handler req)
+          content-type (response/get-header res "Content-Type")]
+      ;; check that the response is html and that the path is for a root page: a single / in the path
+      ;; then we know we don't want it cached (e.g. /index.html)
+      (if (and (= content-type "text/html")
+               (= 1 (count (re-seq #"/" uri))))
+        ;; response for html page in root directory, no-cache
+        (response/header res "Cache-Control" "no-cache")
+        ;; else, carry on
+        res))))
+
 
 (defn split-divide [val divider]
   [(Integer. (int (/ val divider))) (mod val divider)]


[13/13] storm git commit: add STORM-1964 to changelog

Posted by pt...@apache.org.
add STORM-1964 to changelog


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/bb1ea75e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/bb1ea75e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/bb1ea75e

Branch: refs/heads/1.0.x-branch
Commit: bb1ea75e2dab5802c844d6105332c3d4fb564397
Parents: 6c81357
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jul 21 13:26:26 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 13:26:26 2016 -0400

----------------------------------------------------------------------
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/bb1ea75e/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0b7173a..1f15584 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
 ## 1.0.2
+ * STORM-1964: Unexpected behavior when using count window together with timestamp extraction
  * STORM-1977: Restore logic: give up leadership when elected as leader but doesn't have one or more topology codes on local 
  * STORM-1890 ensure we refetch static resources after package build
  * STORM-1730: LocalCluster#shutdown() does not terminate all storm threds/thread pools.


[09/13] storm git commit: add STORM-1964 to changelog

Posted by pt...@apache.org.
add STORM-1964 to changelog


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5e5bb8f1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5e5bb8f1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5e5bb8f1

Branch: refs/heads/master
Commit: 5e5bb8f1ffff56026791563edede427b1ae6d9b1
Parents: a97b162
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jul 21 13:23:00 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 13:23:00 2016 -0400

----------------------------------------------------------------------
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/5e5bb8f1/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f00febd..d660ebc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
 ## 2.0.0
+ * STORM-1964: Unexpected behavior when using count window together with timestamp extraction
  * STORM-1890: ensure we refetch static resources after package build
  * STORM-1966 Expand metric having Map type as value into multiple metrics based on entries
  * STORM-1737: storm-kafka-client has compilation errors with Apache Kafka 0.10


[12/13] storm git commit: Unexpected behavior when using count window together with timestamp extraction

Posted by pt...@apache.org.
Unexpected behavior when using count window together with timestamp extraction

Do not use timestamp to determine the event count in WatermarkCountEvictionPolicy
when count based trigger is used.

Signed-off-by: P. Taylor Goetz <pt...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6c813578
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6c813578
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6c813578

Branch: refs/heads/1.0.x-branch
Commit: 6c813578c07fa856ad930dbd02d192a752f837a0
Parents: b56a34a
Author: Arun Mahadevan <ar...@apache.org>
Authored: Thu Jul 14 00:00:57 2016 +0530
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 13:26:03 2016 -0400

----------------------------------------------------------------------
 .../storm/topology/WindowedBoltExecutor.java    |  2 +-
 .../storm/windowing/CountEvictionPolicy.java    | 12 ++--
 .../storm/windowing/CountTriggerPolicy.java     |  2 +-
 .../storm/windowing/DefaultEvictionContext.java | 53 +++++++++++++++
 .../apache/storm/windowing/EvictionContext.java | 47 +++++++++++++
 .../apache/storm/windowing/EvictionPolicy.java  |  3 +-
 .../storm/windowing/TimeEvictionPolicy.java     |  4 +-
 .../storm/windowing/TimeTriggerPolicy.java      |  2 +-
 .../windowing/WatermarkCountEvictionPolicy.java | 27 +++++---
 .../windowing/WatermarkCountTriggerPolicy.java  |  2 +-
 .../windowing/WatermarkTimeTriggerPolicy.java   |  3 +-
 .../storm/windowing/WindowManagerTest.java      | 69 ++++++++++++++++++--
 12 files changed, 197 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
index 9439ef2..0b18646 100644
--- a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
+++ b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
@@ -238,7 +238,7 @@ public class WindowedBoltExecutor implements IRichBolt {
                                                     WindowManager<Tuple> manager) {
         if (windowLengthCount != null) {
             if (isTupleTs()) {
-                return new WatermarkCountEvictionPolicy<>(windowLengthCount.value, manager);
+                return new WatermarkCountEvictionPolicy<>(windowLengthCount.value);
             } else {
                 return new CountEvictionPolicy<>(windowLengthCount.value);
             }

http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
index 0e7b233..fb12202 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
@@ -17,7 +17,7 @@
  */
 package org.apache.storm.windowing;
 
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * An eviction policy that tracks event counts and can
@@ -26,12 +26,12 @@ import java.util.concurrent.atomic.AtomicInteger;
  * @param <T> the type of event tracked by this policy.
  */
 public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
-    private final int threshold;
-    protected final AtomicInteger currentCount;
+    protected final int threshold;
+    protected final AtomicLong currentCount;
 
     public CountEvictionPolicy(int count) {
         this.threshold = count;
-        this.currentCount = new AtomicInteger();
+        this.currentCount = new AtomicLong();
     }
 
     @Override
@@ -41,7 +41,7 @@ public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
          * return if the event should be evicted
          */
         while (true) {
-            int curVal = currentCount.get();
+            long curVal = currentCount.get();
             if (curVal > threshold) {
                 if (currentCount.compareAndSet(curVal, curVal - 1)) {
                     return Action.EXPIRE;
@@ -61,7 +61,7 @@ public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
     }
 
     @Override
-    public void setContext(Object context) {
+    public void setContext(EvictionContext context) {
         // NOOP
     }
 

http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
index 2594c67..17750b6 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
@@ -44,7 +44,7 @@ public class CountTriggerPolicy<T> implements TriggerPolicy<T> {
     public void track(Event<T> event) {
         if (started && !event.isWatermark()) {
             if (currentCount.incrementAndGet() >= count) {
-                evictionPolicy.setContext(System.currentTimeMillis());
+                evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis()));
                 handler.onTrigger();
             }
         }

http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
new file mode 100644
index 0000000..ef65f66
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.windowing;
+
+public class DefaultEvictionContext implements EvictionContext {
+    private final Long referenceTime;
+    private final Long currentCount;
+    private final Long slidingCount;
+
+    public DefaultEvictionContext(Long referenceTime) {
+        this(referenceTime, null);
+    }
+
+    public DefaultEvictionContext(Long referenceTime, Long currentCount) {
+        this(referenceTime, currentCount, null);
+    }
+
+    public DefaultEvictionContext(Long referenceTime, Long currentCount, Long slidingCount) {
+        this.referenceTime = referenceTime;
+        this.currentCount = currentCount;
+        this.slidingCount = slidingCount;
+    }
+
+    @Override
+    public Long getReferenceTime() {
+        return referenceTime;
+    }
+
+    @Override
+    public Long getCurrentCount() {
+        return currentCount;
+    }
+
+    @Override
+    public Long getSlidingCount() {
+        return slidingCount;
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
new file mode 100644
index 0000000..37dcfd9
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.windowing;
+
+/**
+ * Context information that can be used by the eviction policy
+ */
+public interface EvictionContext {
+    /**
+     * Returns the reference time that the eviction policy could use to
+     * evict the events. In the case of event time processing, this would be
+     * the watermark time.
+     *
+     * @return the reference time in millis
+     */
+    Long getReferenceTime();
+
+    /**
+     * Returns the sliding count for count based windows
+     *
+     * @return the sliding count
+     */
+    Long getSlidingCount();
+
+    /**
+     * Returns the current count of events in the queue up to the reference tim
+     * based on which count based evictions can be performed.
+     *
+     * @return the current count
+     */
+    Long getCurrentCount();
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
index 8128d80..05e4d93 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
@@ -70,5 +70,6 @@ public interface EvictionPolicy<T> {
      *
      * @param context
      */
-    void setContext(Object context);
+    void setContext(EvictionContext context);
+
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
index a2db239..e646207 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
@@ -57,8 +57,8 @@ public class TimeEvictionPolicy<T> implements EvictionPolicy<T> {
     }
 
     @Override
-    public void setContext(Object context) {
-        referenceTime = ((Number) context).longValue();
+    public void setContext(EvictionContext context) {
+        referenceTime = context.getReferenceTime();
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
index 1ad7714..b057afb 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
@@ -115,7 +115,7 @@ public class TimeTriggerPolicy<T> implements TriggerPolicy<T> {
                      * to evict the events
                      */
                     if (evictionPolicy != null) {
-                        evictionPolicy.setContext(System.currentTimeMillis());
+                        evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis()));
                     }
                     handler.onTrigger();
                 } catch (Throwable th) {

http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
index e1af6fb..74240bb 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
@@ -24,25 +24,29 @@ package org.apache.storm.windowing;
  * @param <T> the type of event tracked by this policy.
  */
 public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
-    private final WindowManager<T> windowManager;
     /*
      * The reference time in millis for window calculations and
      * expiring events. If not set it will default to System.currentTimeMillis()
      */
     private long referenceTime;
+    private long processed = 0L;
 
-    public WatermarkCountEvictionPolicy(int count, WindowManager<T> windowManager) {
+    public WatermarkCountEvictionPolicy(int count) {
         super(count);
-        this.windowManager = windowManager;
     }
 
     @Override
     public Action evict(Event<T> event) {
-        if (event.getTimestamp() <= referenceTime) {
-            return super.evict(event);
+        Action action;
+        if (event.getTimestamp() <= referenceTime && processed < currentCount.get()) {
+            action = super.evict(event);
+            if (action == Action.PROCESS) {
+                ++processed;
+            }
         } else {
-            return Action.KEEP;
+            action = Action.KEEP;
         }
+        return action;
     }
 
     @Override
@@ -51,9 +55,14 @@ public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
     }
 
     @Override
-    public void setContext(Object context) {
-        referenceTime = (Long) context;
-        currentCount.set(windowManager.getEventCount(referenceTime));
+    public void setContext(EvictionContext context) {
+        referenceTime = context.getReferenceTime();
+        if (context.getCurrentCount() != null) {
+            currentCount.set(context.getCurrentCount());
+        } else {
+            currentCount.set(processed + context.getSlidingCount());
+        }
+        processed = 0;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
index 391efee..3cfcaad 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
@@ -74,7 +74,7 @@ public class WatermarkCountTriggerPolicy<T> implements TriggerPolicy<T> {
         long watermarkTs = waterMarkEvent.getTimestamp();
         List<Long> eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, count);
         for (long ts : eventTs) {
-            evictionPolicy.setContext(ts);
+            evictionPolicy.setContext(new DefaultEvictionContext(ts, null, Long.valueOf(count)));
             handler.onTrigger();
             lastProcessedTs = ts;
         }

http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
index 3734c93..00620c4 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
@@ -75,7 +75,8 @@ public class WatermarkTimeTriggerPolicy<T> implements TriggerPolicy<T> {
         long windowEndTs = nextWindowEndTs;
         LOG.debug("Window end ts {} Watermark ts {}", windowEndTs, watermarkTs);
         while (windowEndTs <= watermarkTs) {
-            evictionPolicy.setContext(windowEndTs);
+            long currentCount = windowManager.getEventCount(windowEndTs);
+            evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount));
             if (handler.onTrigger()) {
                 windowEndTs += slidingIntervalMs;
             } else {

http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java b/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
index a545427..6645566 100644
--- a/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
@@ -187,7 +187,7 @@ public class WindowManagerTest {
             windowManager.add(i, now - 1000);
         }
         // simulate the time trigger by setting the reference time and invoking onTrigger() manually
-        evictionPolicy.setContext(now + 100);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 100));
         windowManager.onTrigger();
 
         // 100 events with past ts should expire
@@ -210,7 +210,7 @@ public class WindowManagerTest {
         }
         activationsEvents.addAll(newEvents);
         // simulate the time trigger by setting the reference time and invoking onTrigger() manually
-        evictionPolicy.setContext(now + 200);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 200));
         windowManager.onTrigger();
         assertTrue(listener.onExpiryEvents.isEmpty());
         assertEquals(activationsEvents, listener.onActivationEvents);
@@ -236,20 +236,20 @@ public class WindowManagerTest {
             windowManager.add(i);
         }
         // simulate the time trigger by setting the reference time and invoking onTrigger() manually
-        evictionPolicy.setContext(now + 60);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 60));
         windowManager.onTrigger();
 
         assertEquals(seq(1, 10), listener.onActivationEvents);
         assertTrue(listener.onActivationExpiredEvents.isEmpty());
         listener.clear();
         // wait so all events expire
-        evictionPolicy.setContext(now + 120);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 120));
         windowManager.onTrigger();
 
         assertEquals(seq(1, 10), listener.onExpiryEvents);
         assertTrue(listener.onActivationEvents.isEmpty());
         listener.clear();
-        evictionPolicy.setContext(now + 180);
+        evictionPolicy.setContext(new DefaultEvictionContext(now + 180));
         windowManager.onTrigger();
         assertTrue(listener.onActivationExpiredEvents.isEmpty());
         assertTrue(listener.onActivationEvents.isEmpty());
@@ -354,7 +354,7 @@ public class WindowManagerTest {
 
     @Test
     public void testCountBasedWindowWithEventTs() throws Exception {
-        EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(3, windowManager);
+        EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(3);
         windowManager.setEvictionPolicy(evictionPolicy);
         TriggerPolicy<Integer> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager);
         triggerPolicy.start();
@@ -430,7 +430,64 @@ public class WindowManagerTest {
         assertEquals(seq(9), listener.allOnActivationEvents.get(0));
         assertEquals(seq(9, 12), listener.allOnActivationEvents.get(1));
     }
+
+    @Test
+    public void testCountBasedTumblingWithSameEventTs() throws Exception {
+        EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(2);
+        windowManager.setEvictionPolicy(evictionPolicy);
+        TriggerPolicy<Integer> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager);
+        triggerPolicy.start();
+        windowManager.setTriggerPolicy(triggerPolicy);
+
+        windowManager.add(1, 10);
+        windowManager.add(2, 10);
+        windowManager.add(3, 11);
+        windowManager.add(4, 12);
+        windowManager.add(5, 12);
+        windowManager.add(6, 12);
+        windowManager.add(7, 12);
+        windowManager.add(8, 13);
+        windowManager.add(9, 14);
+        windowManager.add(10, 15);
+
+        windowManager.add(new WaterMarkEvent<Integer>(20));
+        assertEquals(5, listener.allOnActivationEvents.size());
+        assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0));
+        assertEquals(seq(3, 4), listener.allOnActivationEvents.get(1));
+        assertEquals(seq(5, 6), listener.allOnActivationEvents.get(2));
+        assertEquals(seq(7, 8), listener.allOnActivationEvents.get(3));
+        assertEquals(seq(9, 10), listener.allOnActivationEvents.get(4));
+    }
+
     @Test
+    public void testCountBasedSlidingWithSameEventTs() throws Exception {
+        EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(5);
+        windowManager.setEvictionPolicy(evictionPolicy);
+        TriggerPolicy<Integer> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager);
+        triggerPolicy.start();
+        windowManager.setTriggerPolicy(triggerPolicy);
+
+        windowManager.add(1, 10);
+        windowManager.add(2, 10);
+        windowManager.add(3, 11);
+        windowManager.add(4, 12);
+        windowManager.add(5, 12);
+        windowManager.add(6, 12);
+        windowManager.add(7, 12);
+        windowManager.add(8, 13);
+        windowManager.add(9, 14);
+        windowManager.add(10, 15);
+
+        windowManager.add(new WaterMarkEvent<Integer>(20));
+        assertEquals(5, listener.allOnActivationEvents.size());
+        assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0));
+        assertEquals(seq(1, 4), listener.allOnActivationEvents.get(1));
+        assertEquals(seq(2, 6), listener.allOnActivationEvents.get(2));
+        assertEquals(seq(4, 8), listener.allOnActivationEvents.get(3));
+        assertEquals(seq(6, 10), listener.allOnActivationEvents.get(4));
+
+    }
+        @Test
     public void testEventTimeLag() throws Exception {
         EvictionPolicy<Integer> evictionPolicy = new WatermarkTimeEvictionPolicy<>(20, 5);
         windowManager.setEvictionPolicy(evictionPolicy);


[05/13] storm git commit: STORM-1890: ensure static UI resources are re-requested with different Storm package builds

Posted by pt...@apache.org.
STORM-1890: ensure static UI resources are re-requested with different Storm package builds

Signed-off-by: P. Taylor Goetz <pt...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/28ef601f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/28ef601f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/28ef601f

Branch: refs/heads/1.0.x-branch
Commit: 28ef601fca4721ec5b285f4aac55dddee783a6df
Parents: f81f30b
Author: Alessandro Bellina <ab...@yahoo-inc.com>
Authored: Tue Jun 14 07:39:45 2016 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 12:14:18 2016 -0400

----------------------------------------------------------------------
 storm-core/src/ui/public/component.html         | 30 ++++++++++----------
 .../src/ui/public/deep_search_result.html       | 10 +++----
 storm-core/src/ui/public/index.html             | 18 ++++++------
 storm-core/src/ui/public/js/script.js           | 12 ++++++++
 storm-core/src/ui/public/js/visualization.js    |  4 +--
 storm-core/src/ui/public/logviewer_search.html  |  6 ++--
 storm-core/src/ui/public/search_result.html     |  6 ++--
 storm-core/src/ui/public/topology.html          | 10 +++----
 storm-dist/binary/pom.xml                       | 10 +++++++
 storm-dist/binary/src/main/assembly/binary.xml  | 25 +++++++++++++++-
 10 files changed, 88 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/component.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/component.html b/storm-core/src/ui/public/component.html
index 37ee525..fc7b15e 100644
--- a/storm-core/src/ui/public/component.html
+++ b/storm-core/src/ui/public/component.html
@@ -22,7 +22,7 @@
 <link href="/css/bootstrap-3.3.1.min.css" rel="stylesheet" type="text/css">
 <link href="/css/jquery.dataTables.1.10.4.min.css" rel="stylesheet" type="text/css">
 <link href="/css/dataTables.bootstrap.css" rel="stylesheet" type="text/css">
-<link href="/css/style.css" rel="stylesheet" type="text/css">
+<link href="/css/style.css?_ts=${packageTimestamp}" rel="stylesheet" type="text/css">
 <script src="/js/jquery-1.11.1.min.js" type="text/javascript"></script>
 <script src="/js/jquery.dataTables.1.10.4.min.js" type="text/javascript"></script>
 <script src="/js/jquery.cookies.2.2.0.min.js" type="text/javascript"></script>
@@ -32,7 +32,7 @@
 <script src="/js/jquery.blockUI.min.js" type="text/javascript"></script>
 <script src="/js/moment.min.js" type="text/javascript"></script>
 <script src="/js/dataTables.bootstrap.min.js" type="text/javascript"></script>
-<script src="/js/script.js" type="text/javascript"></script>
+<script src="/js/script.js?_ts=${packageTimestamp}" type="text/javascript"></script>
 </head>
 <body>
 <div class="container-fluid">
@@ -83,7 +83,7 @@ function jsError(other) {
     try {
       other();
     } catch (err) {
-      $.get("/templates/json-error-template.html", function(template) {
+      getStatic("/templates/json-error-template.html", function(template) {
         $("#json-response-error").append(Mustache.render($(template).filter("#json-error-template").html(),{error: "JS Error", errorMessage: err}));
       });
     }
@@ -156,7 +156,7 @@ $(document).ready(function() {
     $.ajaxSetup({
         "error":function(jqXHR,textStatus,response) {
             var errorJson = jQuery.parseJSON(jqXHR.responseText);
-            $.get("/templates/json-error-template.html", function(template) {
+            getStatic("/templates/json-error-template.html", function(template) {
                 $("#json-response-error").append(Mustache.render($(template).filter("#json-error-template").html(),errorJson));
             });
         }
@@ -187,7 +187,7 @@ $(document).ready(function() {
 
     $.getJSON(url,function(response,status,jqXHR) {
         var uiUser = $("#ui-user");
-        $.get("/templates/user-template.html", function(template) {
+        getStatic("/templates/user-template.html", function(template) {
             uiUser.append(Mustache.render($(template).filter("#user-template").html(),response));
             $('#ui-user [data-toggle="tooltip"]').tooltip()
         });
@@ -218,7 +218,7 @@ $(document).ready(function() {
         var profilerControl = $("#profiler-control");
         var executorStats = $("#component-executor-stats");
         var componentErrors = $("#component-errors");
-        $.get("/templates/component-page-template.html", function(template) {
+        getStatic("/templates/component-page-template.html", function(template) {
             response["profilerActive"] = $.map(response["profilerActive"], function(active_map) {
                 var date = new Date();
                 var millis = date.getTime() + parseInt(active_map["timestamp"]);
@@ -357,9 +357,9 @@ function start_profiling() {
     var passed = {}
     Object.keys(workerActionSelected).forEach(function (id) {
         var url = "/api/v1/topology/"+topologyId+"/profiling/start/" + id + "/" + timeout;
-        $.get(url, function(response,status,jqXHR) {
+        getStatic(url, function(response,status,jqXHR) {
             jsError(function() {
-                $.get("/templates/component-page-template.html", function(template) {
+                getStatic("/templates/component-page-template.html", function(template) {
                     var host_port_split = id.split(":");
                     var host = host_port_split[0];
                     var port = host_port_split[1];
@@ -398,7 +398,7 @@ function stop_profiling(id) {
     $("#stop_" + id).prop('disabled', true);
     setTimeout(function(){ $("#stop_" + id).prop('disabled', false); }, 5000);
     
-    $.get(url, function(response,status,jqXHR) {
+    getStatic(url, function(response,status,jqXHR) {
         alert("Submitted request to stop profiling...");
     })
     .fail(function(response) {
@@ -414,7 +414,7 @@ function dump_profile(id) {
     $("#dump_profile_" + id).prop('disabled', true);
     setTimeout(function(){ $("#dump_profile_" + id).prop('disabled', false); }, 5000);
     
-    $.get(url, function(response,status,jqXHR) {
+    getStatic(url, function(response,status,jqXHR) {
         alert("Submitted request to dump profile snapshot...");
     })
     .fail(function(response) {
@@ -434,7 +434,7 @@ function dump_jstacks() {
         $("#dump_jstack_" + id).prop('disabled', true);
         setTimeout(function(){ $("#dump_jstack_" + id).prop('disabled', false); }, 5000);
 
-        $.get(url)
+        getStatic(url)
         .fail(function(response) {
             failed[id] = response;
         });
@@ -459,7 +459,7 @@ function dump_jstack(id) {
     $("#dump_jstack_" + id).prop('disabled', true);
     setTimeout(function(){ $("#dump_jstack_" + id).prop('disabled', false); }, 5000);
     
-    $.get(url, function(response,status,jqXHR) {
+    getStatic(url, function(response,status,jqXHR) {
         alert("Submitted request for jstack dump...");
     })
     .fail(function(response) {
@@ -477,7 +477,7 @@ function restart_worker_jvms() {
         $("#restart_worker_jvm_" + id).prop('disabled', true);
         setTimeout(function(){ $("#restart_worker_jvm_" + id).prop('disabled', false); }, 5000);
 
-        $.get(url)
+        getStatic(url)
         .fail(function(response) {
             failed[id] = response;
         });
@@ -505,7 +505,7 @@ function dump_heaps() {
         $("#dump_heap_" + id).prop('disabled', true);
         setTimeout(function(){ $("#dump_heap_" + id).prop('disabled', false); }, 5000);
 
-        $.get(url)
+        getStatic(url)
         .fail(function(response) {
             failed[id] = response;
         });
@@ -530,7 +530,7 @@ function dump_heap(id) {
     $("#dump_heap_" + id).prop('disabled', true);
     setTimeout(function(){ $("#dump_heap_" + id).prop('disabled', false); }, 5000);
     
-    $.get(url, function(response,status,jqXHR) {
+    getStatic(url, function(response,status,jqXHR) {
         alert("Submitted request for jmap dump...");
     })
     .fail(function(response) {

http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/deep_search_result.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/deep_search_result.html b/storm-core/src/ui/public/deep_search_result.html
index 406a101..5b4f47b 100644
--- a/storm-core/src/ui/public/deep_search_result.html
+++ b/storm-core/src/ui/public/deep_search_result.html
@@ -21,7 +21,7 @@
 <link href="/css/bootstrap-3.3.1.min.css" rel="stylesheet" type="text/css">
 <link href="/css/jquery.dataTables.1.10.4.min.css" rel="stylesheet" type="text/css">
 <link href="/css/dataTables.bootstrap.css" rel="stylesheet" type="text/css">
-<link href="/css/style.css" rel="stylesheet" type="text/css">
+<link href="/css/style.css?_ts=${packageTimestamp}" rel="stylesheet" type="text/css">
 <script src="/js/jquery-1.11.1.min.js" type="text/javascript"></script>
 <script src="/js/jquery.dataTables.1.10.4.min.js" type="text/javascript"></script>
 <script src="/js/jquery.cookies.2.2.0.min.js" type="text/javascript"></script>
@@ -31,7 +31,7 @@
 <script src="/js/jquery.blockUI.min.js" type="text/javascript"></script>
 <script src="/js/url.min.js" type="text/javascript"></script>
 <script src="/js/dataTables.bootstrap.min.js" type="text/javascript"></script>
-<script src="/js/script.js" type="text/javascript"></script>
+<script src="/js/script.js?_ts=${packageTimestamp}" type="text/javascript"></script>
 </head>
 <body>
 <div class="container-fluid">
@@ -56,12 +56,12 @@ $(document).ready(function() {
     var port = $.url("?port") || "*";
     var search_archived = $.url("?search-archived");
 
-    $.get("/templates/deep-search-result-page-template.html", function(template) {
+    getStatic("/templates/deep-search-result-page-template.html", function(template) {
         var checked;
         if (search_archived) {
             checked = "checked";
         }
-        $.get("/api/v1/history/summary", function (response, status, jqXHR) {
+        getStatic("/api/v1/history/summary", function (response, status, jqXHR) {
             var findIds = function findIds(query, cb) {
                 var found = [];
                 var re = new RegExp(query, 'i');
@@ -104,7 +104,7 @@ $(document).ready(function() {
 
         var result = $("#result");
         if(search) {
-            $.get("/api/v1/supervisor/summary", function(response, status, jqXHR) {
+            getStatic("/api/v1/supervisor/summary", function(response, status, jqXHR) {
 
                 for(var i in response.supervisors) {
                     response.supervisors[i].elemId = elem_id_for_host(response.supervisors[i].host);

http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/index.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/index.html b/storm-core/src/ui/public/index.html
index eae423c..52900a5 100644
--- a/storm-core/src/ui/public/index.html
+++ b/storm-core/src/ui/public/index.html
@@ -23,7 +23,7 @@
 <link href="/css/jquery.dataTables.1.10.4.min.css" rel="stylesheet" type="text/css">
 <link href="/css/dataTables.bootstrap.css" rel="stylesheet" type="text/css">
 <link href="/css/jsonFormatter.min.css" rel="stylesheet" type="text/css">
-<link href="/css/style.css" rel="stylesheet" type="text/css">
+<link href="/css/style.css?_ts=${packageTimestamp}" rel="stylesheet" type="text/css">
 <script src="/js/jquery-1.11.1.min.js" type="text/javascript"></script>
 <script src="/js/jquery.dataTables.1.10.4.min.js" type="text/javascript"></script>
 <script src="/js/jquery.cookies.2.2.0.min.js" type="text/javascript"></script>
@@ -32,7 +32,7 @@
 <script src="/js/jquery.blockUI.min.js" type="text/javascript"></script>
 <script src="/js/dataTables.bootstrap.min.js" type="text/javascript"></script>
 <script src="/js/jsonFormatter.min.js" type="text/javascript"></script>
-<script src="/js/script.js" type="text/javascript"></script>
+<script src="/js/script.js?_ts=${packageTimestamp}" type="text/javascript"></script>
 </head>
 <body>
 <div class="container-fluid">
@@ -96,7 +96,7 @@ $(document).ready(function() {
     $.ajaxSetup({
         "error":function(jqXHR,textStatus,response) {
             var errorJson = jQuery.parseJSON(jqXHR.responseText);
-            $.get("/templates/json-error-template.html", function(template) {
+            getStatic("/templates/json-error-template.html", function(template) {
                 $("#json-response-error").append(Mustache.render($(template).filter("#json-error-template").html(),errorJson));
             });
         }
@@ -109,18 +109,18 @@ $(document).ready(function() {
     var config = $("#nimbus-configuration");
 
     $.getJSON("/api/v1/cluster/summary",function(response,status,jqXHR) {
-        $.get("/templates/user-template.html", function(template) {
+        getStatic("/templates/user-template.html", function(template) {
             uiUser.append(Mustache.render($(template).filter("#user-template").html(),response));
             $('#ui-user [data-toggle="tooltip"]').tooltip()
         });
 
-        $.get("/templates/index-page-template.html", function(template) {
+        getStatic("/templates/index-page-template.html", function(template) {
             clusterSummary.append(Mustache.render($(template).filter("#cluster-summary-template").html(),response));
             $('#cluster-summary [data-toggle="tooltip"]').tooltip();
         });
     });
     $.getJSON("/api/v1/nimbus/summary",function(response,status,jqXHR) {
-      $.get("/templates/index-page-template.html", function(template) {
+      getStatic("/templates/index-page-template.html", function(template) {
           nimbusSummary.append(Mustache.render($(template).filter("#nimbus-summary-template").html(),response));
           //host, port, isLeader, version, uptime
           dtAutoPage("#nimbus-summary-table", {
@@ -133,7 +133,7 @@ $(document).ready(function() {
       });
     });
     $.getJSON("/api/v1/topology/summary",function(response,status,jqXHR) {
-      $.get("/templates/index-page-template.html", function(template) {
+      getStatic("/templates/index-page-template.html", function(template) {
           topologySummary.append(Mustache.render($(template).filter("#topology-summary-template").html(),response));
           //name, owner, status, uptime, num workers, num executors, num tasks, replication count, assigned total mem, assigned total cpu, scheduler info
           dtAutoPage("#topology-summary-table", {
@@ -146,7 +146,7 @@ $(document).ready(function() {
       });
     });
     $.getJSON("/api/v1/supervisor/summary",function(response,status,jqXHR) {
-      $.get("/templates/index-page-template.html", function(template) {
+      getStatic("/templates/index-page-template.html", function(template) {
           supervisorSummary.append(Mustache.render($(template).filter("#supervisor-summary-template").html(),response));
           //id, host, uptime, slots, used slots
           dtAutoPage("#supervisor-summary-table", {
@@ -160,7 +160,7 @@ $(document).ready(function() {
     });
     $.getJSON("/api/v1/cluster/configuration",function(response,status,jqXHR) {
       var formattedResponse = formatConfigData(response);
-      $.get("/templates/index-page-template.html", function(template) {
+      getStatic("/templates/index-page-template.html", function(template) {
         config.append(Mustache.render($(template).filter("#configuration-template").html(),formattedResponse));
         $('#nimbus-configuration-table td').jsonFormatter()
         //key, value

http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/js/script.js
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/js/script.js b/storm-core/src/ui/public/js/script.js
index ff3c525..769442f 100644
--- a/storm-core/src/ui/public/js/script.js
+++ b/storm-core/src/ui/public/js/script.js
@@ -259,3 +259,15 @@ $.blockUI.defaults.css = {
     opacity: .5,
     color: '#fff',margin:0,width:"30%",top:"40%",left:"35%",textAlign:"center"
 };
+
+// add a url query param to static (templates normally) ajax requests
+// for cache busting
+function getStatic(url, cb) {
+    return $.ajax({
+        url: url,
+        data: {
+            '_ts': '${packageTimestamp}'
+        },
+        success: cb
+    });
+};

http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/js/visualization.js
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/js/visualization.js b/storm-core/src/ui/public/js/visualization.js
index 9aab087..4daadc2 100644
--- a/storm-core/src/ui/public/js/visualization.js
+++ b/storm-core/src/ui/public/js/visualization.js
@@ -379,7 +379,7 @@ function jsError(other) {
   try {
     other();
   } catch (err) {
-    $.get("/templates/json-error-template.html", function(template) {
+    getStatic("/templates/json-error-template.html", function(template) {
       $("#json-response-error").append(Mustache.render($(template).filter("#json-error-template").html(),{error: "JS Error", errorMessage: err}));
     });
   }
@@ -388,7 +388,7 @@ function jsError(other) {
 var should_update;
 function show_visualization(sys) {
     $.getJSON("/api/v1/topology/"+$.url("?id")+"/visualization-init",function(response,status,jqXHR) {
-        $.get("/templates/topology-page-template.html", function(template) {
+        getStatic("/templates/topology-page-template.html", function(template) {
             jsError(function() {
                 var topologyVisualization = $("#visualization-container");
                 topologyVisualization.append(

http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/logviewer_search.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/logviewer_search.html b/storm-core/src/ui/public/logviewer_search.html
index f981499..3a1682e 100644
--- a/storm-core/src/ui/public/logviewer_search.html
+++ b/storm-core/src/ui/public/logviewer_search.html
@@ -21,7 +21,7 @@
 <link href="/css/bootstrap-3.3.1.min.css" rel="stylesheet" type="text/css">
 <link href="/css/jquery.dataTables.1.10.4.min.css" rel="stylesheet" type="text/css">
 <link href="/css/dataTables.bootstrap.css" rel="stylesheet" type="text/css">
-<link href="/css/style.css" rel="stylesheet" type="text/css">
+<link href="/css/style.css?_ts=${packageTimestamp}" rel="stylesheet" type="text/css">
 <script src="/js/jquery-1.11.1.min.js" type="text/javascript"></script>
 <script src="/js/jquery.dataTables.1.10.4.min.js" type="text/javascript"></script>
 <script src="/js/jquery.cookies.2.2.0.min.js" type="text/javascript"></script>
@@ -30,7 +30,7 @@
 <script src="/js/jquery.blockUI.min.js" type="text/javascript"></script>
 <script src="/js/url.min.js" type="text/javascript"></script>
 <script src="/js/dataTables.bootstrap.min.js" type="text/javascript"></script>
-<script src="/js/script.js" type="text/javascript"></script>
+<script src="/js/script.js?_ts=${packageTimestamp}" type="text/javascript"></script>
 </head>
 <body>
 <div class="container-fluid">
@@ -51,7 +51,7 @@ $(document).ready(function() {
     file = decodeURIComponent(file);
     search = decodeURIComponent(search);
 
-    $.get("/templates/logviewer-search-page-template.html", function(template) {
+    getStatic("/templates/logviewer-search-page-template.html", function(template) {
         $("#search-form").append(Mustache.render($(template).filter("#search-single-file").html(),{file: file, search: search, isDaemon: isDaemon}));
 
         var result = $("#result");

http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/search_result.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/search_result.html b/storm-core/src/ui/public/search_result.html
index ec44aa9..ee0efd8 100644
--- a/storm-core/src/ui/public/search_result.html
+++ b/storm-core/src/ui/public/search_result.html
@@ -21,7 +21,7 @@
 <link href="/css/bootstrap-3.3.1.min.css" rel="stylesheet" type="text/css">
 <link href="/css/jquery.dataTables.1.10.4.min.css" rel="stylesheet" type="text/css">
 <link href="/css/dataTables.bootstrap.css" rel="stylesheet" type="text/css">
-<link href="/css/style.css" rel="stylesheet" type="text/css">
+<link href="/css/style.css?_ts=${packageTimestamp}" rel="stylesheet" type="text/css">
 <script src="/js/jquery-1.11.1.min.js" type="text/javascript"></script>
 <script src="/js/jquery.dataTables.1.10.4.min.js" type="text/javascript"></script>
 <script src="/js/jquery.cookies.2.2.0.min.js" type="text/javascript"></script>
@@ -30,7 +30,7 @@
 <script src="/js/jquery.blockUI.min.js" type="text/javascript"></script>
 <script src="/js/url.min.js" type="text/javascript"></script>
 <script src="/js/dataTables.bootstrap.min.js" type="text/javascript"></script>
-<script src="/js/script.js" type="text/javascript"></script>
+<script src="/js/script.js?_ts=${packageTimestamp}" type="text/javascript"></script>
 </head>
 <body>
 <div class="container-fluid">
@@ -49,7 +49,7 @@ $(document).ready(function() {
     var count = $.url("?count") || 2;
     var searchArchived = $.url("?searchArchived") || "";
 
-    $.get("/templates/search-result-page-template.html", function(template) {
+    getStatic("/templates/search-result-page-template.html", function(template) {
         $("#search-form").append(Mustache.render($(template).filter("#search-form-template").html(),{id: id, search: search, count: count, searchArchived: searchArchived}));
 
         var result = $("#result");

http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/topology.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/topology.html b/storm-core/src/ui/public/topology.html
index 061fd91..f955165 100644
--- a/storm-core/src/ui/public/topology.html
+++ b/storm-core/src/ui/public/topology.html
@@ -24,7 +24,7 @@
 <link href="/css/jquery.dataTables.1.10.4.min.css" rel="stylesheet" type="text/css">
 <link href="/css/dataTables.bootstrap.css" rel="stylesheet" type="text/css">
 <link href="/css/jsonFormatter.min.css" rel="stylesheet" type="text/css">
-<link href="/css/style.css" rel="stylesheet" type="text/css">
+<link href="/css/style.css?_ts=${packageTimestamp}" rel="stylesheet" type="text/css">
 <script src="/js/jquery-1.11.1.min.js" type="text/javascript"></script>
 <script src="/js/jquery.dataTables.1.10.4.min.js" type="text/javascript"></script>
 <script src="/js/jquery.cookies.2.2.0.min.js" type="text/javascript"></script>
@@ -33,7 +33,7 @@
 <script src="/js/bootstrap-3.3.1.min.js" type="text/javascript"></script>
 <script src="/js/jquery.blockUI.min.js" type="text/javascript"></script>
 <script src="/js/jsonFormatter.min.js" type="text/javascript"></script>
-<script src="/js/script.js" type="text/javascript"></script>
+<script src="/js/script.js?_ts=${packageTimestamp}" type="text/javascript"></script>
 <script src="/js/visualization.js" type="text/javascript"></script>
 <script src="/js/arbor.js" type="text/javascript"></script>
 <script src="/js/arbor-graphics.js" type="text/javascript"></script>
@@ -227,7 +227,7 @@ function renderLogLevelForm (template, responseData){
     };
     if (!responseData) {
         var topologyId = $.url("?id");
-        $.get ('/api/v1/topology/' + topologyId + '/logconfig', renderImpl);
+        getStatic ('/api/v1/topology/' + topologyId + '/logconfig', renderImpl);
     } else {
         renderImpl (responseData);
     }
@@ -261,7 +261,7 @@ $(document).ready(function() {
     $.ajaxSetup({
         "error":function(jqXHR,textStatus,response) {
             var errorJson = jQuery.parseJSON(jqXHR.responseText);
-            $.get("/templates/json-error-template.html", function(template) {
+            getStatic("/templates/json-error-template.html", function(template) {
                 $("#json-response-error").append(Mustache.render($(template).filter("#json-error-template").html(),errorJson));
             });
         }
@@ -269,7 +269,7 @@ $(document).ready(function() {
 
     $.getJSON(url,function(response,status,jqXHR) {
         var uiUser = $("#ui-user");
-        $.get("/templates/user-template.html", function(template) {
+        getStatic("/templates/user-template.html", function(template) {
             uiUser.append(Mustache.render($(template).filter("#user-template").html(),response));
             $('#ui-user [data-toggle="tooltip"]').tooltip();
         });

http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-dist/binary/pom.xml
----------------------------------------------------------------------
diff --git a/storm-dist/binary/pom.xml b/storm-dist/binary/pom.xml
index a26f855..f7c4cdb 100644
--- a/storm-dist/binary/pom.xml
+++ b/storm-dist/binary/pom.xml
@@ -30,6 +30,16 @@
     <name>Storm Binary Distribution</name>
     <description>Storm binary distribution</description>
 
+    <!--
+        Used for cache busting in the Storm UI HTML and script.js files
+        See src/main/assembly/pom.xml for the fileSet rules with
+            <filtered>true</filtered>
+    -->
+    <properties>
+        <packageTimestamp>${maven.build.timestamp}</packageTimestamp>
+        <maven.build.timestamp.format>YYYYMMddHHmm</maven.build.timestamp.format>
+    </properties>
+
     <dependencies>
         <dependency>
             <groupId>org.apache.storm</groupId>

http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-dist/binary/src/main/assembly/binary.xml
----------------------------------------------------------------------
diff --git a/storm-dist/binary/src/main/assembly/binary.xml b/storm-dist/binary/src/main/assembly/binary.xml
index d524530..2333168 100644
--- a/storm-dist/binary/src/main/assembly/binary.xml
+++ b/storm-dist/binary/src/main/assembly/binary.xml
@@ -46,13 +46,36 @@
             </includes>
             <fileMode>0755</fileMode>
         </fileSet>
+        <!--
+            Allow for variable substitution for cache busting in HTML files and script.js
+            See storm-dist/binary/pom.xml for custom variables that will be substituted in.
+
+            The source files should have a ${variable to be replaced} wherever
+            maven assembly plugin should inject a value.
+        -->
         <fileSet>
             <directory>${project.basedir}/../../storm-core/src/ui/public</directory>
             <outputDirectory>public</outputDirectory>
             <includes>
-                <include>*/**</include>
+                <include>*.html</include>
+                <include>js/script.js</include>
             </includes>
             <excludes/>
+            <filtered>true</filtered>
+        </fileSet>
+        <!--
+            Include rest of public/ directory, without any filtering
+        -->
+        <fileSet>
+            <directory>${project.basedir}/../../storm-core/src/ui/public</directory>
+            <outputDirectory>public</outputDirectory>
+            <includes>
+                <include>*/**</include>
+            </includes>
+            <excludes>
+                <exclude>*.html</exclude>
+                <exclude>js/script.js</exclude>
+            </excludes>
         </fileSet>
         <fileSet>
             <directory>${project.basedir}/../../examples</directory>


[08/13] storm git commit: Merge branch 'STORM-1964' of github.com:arunmahadevan/storm

Posted by pt...@apache.org.
Merge branch 'STORM-1964' of github.com:arunmahadevan/storm


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a97b1628
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a97b1628
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a97b1628

Branch: refs/heads/master
Commit: a97b16285afe6a12f63360769985812fe7b5526f
Parents: e5be986 3cbd26e
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jul 21 13:21:24 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 13:21:24 2016 -0400

----------------------------------------------------------------------
 .../storm/topology/WindowedBoltExecutor.java    |  2 +-
 .../storm/windowing/CountEvictionPolicy.java    | 12 ++--
 .../storm/windowing/CountTriggerPolicy.java     |  2 +-
 .../storm/windowing/DefaultEvictionContext.java | 53 +++++++++++++++
 .../apache/storm/windowing/EvictionContext.java | 47 +++++++++++++
 .../apache/storm/windowing/EvictionPolicy.java  |  3 +-
 .../storm/windowing/TimeEvictionPolicy.java     |  4 +-
 .../storm/windowing/TimeTriggerPolicy.java      |  2 +-
 .../windowing/WatermarkCountEvictionPolicy.java | 27 +++++---
 .../windowing/WatermarkCountTriggerPolicy.java  |  2 +-
 .../windowing/WatermarkTimeTriggerPolicy.java   |  3 +-
 .../storm/windowing/WindowManagerTest.java      | 69 ++++++++++++++++++--
 12 files changed, 197 insertions(+), 29 deletions(-)
----------------------------------------------------------------------



[07/13] storm git commit: Merge branch '1.0.x-branch' of https://git-wip-us.apache.org/repos/asf/storm into 1.0.x-branch

Posted by pt...@apache.org.
Merge branch '1.0.x-branch' of https://git-wip-us.apache.org/repos/asf/storm into 1.0.x-branch


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b56a34ab
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b56a34ab
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b56a34ab

Branch: refs/heads/1.0.x-branch
Commit: b56a34ab28f6849503d386f5d2e7b97ffb7f9603
Parents: b068dde 7936de9
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jul 21 12:25:33 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 12:25:33 2016 -0400

----------------------------------------------------------------------
 CHANGELOG.md                                    |  3 ++
 .../apache/storm/command/shell_submission.clj   |  3 +-
 .../src/clj/org/apache/storm/daemon/nimbus.clj  | 21 +++--------
 .../src/clj/org/apache/storm/zookeeper.clj      | 37 +++++++++++++++-----
 .../org/apache/storm/utils/DisruptorQueue.java  | 16 +++++++--
 .../test/clj/org/apache/storm/nimbus_test.clj   | 29 ---------------
 6 files changed, 51 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/b56a34ab/CHANGELOG.md
----------------------------------------------------------------------
diff --cc CHANGELOG.md
index 4182f1a,af5fcff..0b7173a
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@@ -1,5 -1,7 +1,8 @@@
  ## 1.0.2
+  * STORM-1977: Restore logic: give up leadership when elected as leader but doesn't have one or more topology codes on local 
 + * STORM-1890 ensure we refetch static resources after package build
+  * STORM-1730: LocalCluster#shutdown() does not terminate all storm threds/thread pools.
+  * STORM-1976: STORM-1976 Remove cleanup-corrupt-topologies!
   * STORM-1939: Frequent InterruptedException raised by ShellBoltMessageQueue.poll
   * STORM-1968: Storm logviewer does not work for nimbus.log in secure cluster
   * STORM-1928: ShellSpout should check heartbeat while ShellSpout is waiting for subprocess to sync