You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2016/07/21 17:27:27 UTC
[01/13] storm git commit: [STORM-1964] Unexpected behavior when using
count window together with timestamp extraction
Repository: storm
Updated Branches:
refs/heads/1.0.x-branch 7936de9f1 -> bb1ea75e2
refs/heads/1.x-branch 88e8f17d4 -> 6deb60fb4
refs/heads/master e5be9863c -> 5e5bb8f1f
[STORM-1964] Unexpected behavior when using count window together with timestamp extraction
Do not use timestamp to determine the event count in WatermarkCountEvictionPolicy
when count based trigger is used.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3cbd26ec
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3cbd26ec
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3cbd26ec
Branch: refs/heads/master
Commit: 3cbd26ec275bb052195a038b49853ed1cfc84806
Parents: 05e3289
Author: Arun Mahadevan <ar...@apache.org>
Authored: Thu Jul 14 00:00:57 2016 +0530
Committer: Arun Mahadevan <ar...@apache.org>
Committed: Fri Jul 15 11:59:48 2016 +0530
----------------------------------------------------------------------
.../storm/topology/WindowedBoltExecutor.java | 2 +-
.../storm/windowing/CountEvictionPolicy.java | 12 ++--
.../storm/windowing/CountTriggerPolicy.java | 2 +-
.../storm/windowing/DefaultEvictionContext.java | 53 +++++++++++++++
.../apache/storm/windowing/EvictionContext.java | 47 +++++++++++++
.../apache/storm/windowing/EvictionPolicy.java | 3 +-
.../storm/windowing/TimeEvictionPolicy.java | 4 +-
.../storm/windowing/TimeTriggerPolicy.java | 2 +-
.../windowing/WatermarkCountEvictionPolicy.java | 27 +++++---
.../windowing/WatermarkCountTriggerPolicy.java | 2 +-
.../windowing/WatermarkTimeTriggerPolicy.java | 3 +-
.../storm/windowing/WindowManagerTest.java | 69 ++++++++++++++++++--
12 files changed, 197 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
index fe0f4da..3f245e1 100644
--- a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
+++ b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
@@ -253,7 +253,7 @@ public class WindowedBoltExecutor implements IRichBolt {
WindowManager<Tuple> manager) {
if (windowLengthCount != null) {
if (isTupleTs()) {
- return new WatermarkCountEvictionPolicy<>(windowLengthCount.value, manager);
+ return new WatermarkCountEvictionPolicy<>(windowLengthCount.value);
} else {
return new CountEvictionPolicy<>(windowLengthCount.value);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
index 0e7b233..fb12202 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
@@ -17,7 +17,7 @@
*/
package org.apache.storm.windowing;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
/**
* An eviction policy that tracks event counts and can
@@ -26,12 +26,12 @@ import java.util.concurrent.atomic.AtomicInteger;
* @param <T> the type of event tracked by this policy.
*/
public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
- private final int threshold;
- protected final AtomicInteger currentCount;
+ protected final int threshold;
+ protected final AtomicLong currentCount;
public CountEvictionPolicy(int count) {
this.threshold = count;
- this.currentCount = new AtomicInteger();
+ this.currentCount = new AtomicLong();
}
@Override
@@ -41,7 +41,7 @@ public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
* return if the event should be evicted
*/
while (true) {
- int curVal = currentCount.get();
+ long curVal = currentCount.get();
if (curVal > threshold) {
if (currentCount.compareAndSet(curVal, curVal - 1)) {
return Action.EXPIRE;
@@ -61,7 +61,7 @@ public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
}
@Override
- public void setContext(Object context) {
+ public void setContext(EvictionContext context) {
// NOOP
}
http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
index 2594c67..17750b6 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
@@ -44,7 +44,7 @@ public class CountTriggerPolicy<T> implements TriggerPolicy<T> {
public void track(Event<T> event) {
if (started && !event.isWatermark()) {
if (currentCount.incrementAndGet() >= count) {
- evictionPolicy.setContext(System.currentTimeMillis());
+ evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis()));
handler.onTrigger();
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
new file mode 100644
index 0000000..ef65f66
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.windowing;
+
+public class DefaultEvictionContext implements EvictionContext {
+ private final Long referenceTime;
+ private final Long currentCount;
+ private final Long slidingCount;
+
+ public DefaultEvictionContext(Long referenceTime) {
+ this(referenceTime, null);
+ }
+
+ public DefaultEvictionContext(Long referenceTime, Long currentCount) {
+ this(referenceTime, currentCount, null);
+ }
+
+ public DefaultEvictionContext(Long referenceTime, Long currentCount, Long slidingCount) {
+ this.referenceTime = referenceTime;
+ this.currentCount = currentCount;
+ this.slidingCount = slidingCount;
+ }
+
+ @Override
+ public Long getReferenceTime() {
+ return referenceTime;
+ }
+
+ @Override
+ public Long getCurrentCount() {
+ return currentCount;
+ }
+
+ @Override
+ public Long getSlidingCount() {
+ return slidingCount;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
new file mode 100644
index 0000000..37dcfd9
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.windowing;
+
+/**
+ * Context information that can be used by the eviction policy
+ */
+public interface EvictionContext {
+ /**
+ * Returns the reference time that the eviction policy could use to
+ * evict the events. In the case of event time processing, this would be
+ * the watermark time.
+ *
+ * @return the reference time in millis
+ */
+ Long getReferenceTime();
+
+ /**
+ * Returns the sliding count for count based windows
+ *
+ * @return the sliding count
+ */
+ Long getSlidingCount();
+
+ /**
+ * Returns the current count of events in the queue up to the reference tim
+ * based on which count based evictions can be performed.
+ *
+ * @return the current count
+ */
+ Long getCurrentCount();
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
index 8128d80..05e4d93 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
@@ -70,5 +70,6 @@ public interface EvictionPolicy<T> {
*
* @param context
*/
- void setContext(Object context);
+ void setContext(EvictionContext context);
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
index a2db239..e646207 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
@@ -57,8 +57,8 @@ public class TimeEvictionPolicy<T> implements EvictionPolicy<T> {
}
@Override
- public void setContext(Object context) {
- referenceTime = ((Number) context).longValue();
+ public void setContext(EvictionContext context) {
+ referenceTime = context.getReferenceTime();
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
index 1ad7714..b057afb 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
@@ -115,7 +115,7 @@ public class TimeTriggerPolicy<T> implements TriggerPolicy<T> {
* to evict the events
*/
if (evictionPolicy != null) {
- evictionPolicy.setContext(System.currentTimeMillis());
+ evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis()));
}
handler.onTrigger();
} catch (Throwable th) {
http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
index e1af6fb..74240bb 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
@@ -24,25 +24,29 @@ package org.apache.storm.windowing;
* @param <T> the type of event tracked by this policy.
*/
public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
- private final WindowManager<T> windowManager;
/*
* The reference time in millis for window calculations and
* expiring events. If not set it will default to System.currentTimeMillis()
*/
private long referenceTime;
+ private long processed = 0L;
- public WatermarkCountEvictionPolicy(int count, WindowManager<T> windowManager) {
+ public WatermarkCountEvictionPolicy(int count) {
super(count);
- this.windowManager = windowManager;
}
@Override
public Action evict(Event<T> event) {
- if (event.getTimestamp() <= referenceTime) {
- return super.evict(event);
+ Action action;
+ if (event.getTimestamp() <= referenceTime && processed < currentCount.get()) {
+ action = super.evict(event);
+ if (action == Action.PROCESS) {
+ ++processed;
+ }
} else {
- return Action.KEEP;
+ action = Action.KEEP;
}
+ return action;
}
@Override
@@ -51,9 +55,14 @@ public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
}
@Override
- public void setContext(Object context) {
- referenceTime = (Long) context;
- currentCount.set(windowManager.getEventCount(referenceTime));
+ public void setContext(EvictionContext context) {
+ referenceTime = context.getReferenceTime();
+ if (context.getCurrentCount() != null) {
+ currentCount.set(context.getCurrentCount());
+ } else {
+ currentCount.set(processed + context.getSlidingCount());
+ }
+ processed = 0;
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
index 391efee..3cfcaad 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
@@ -74,7 +74,7 @@ public class WatermarkCountTriggerPolicy<T> implements TriggerPolicy<T> {
long watermarkTs = waterMarkEvent.getTimestamp();
List<Long> eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, count);
for (long ts : eventTs) {
- evictionPolicy.setContext(ts);
+ evictionPolicy.setContext(new DefaultEvictionContext(ts, null, Long.valueOf(count)));
handler.onTrigger();
lastProcessedTs = ts;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
index 3734c93..00620c4 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
@@ -75,7 +75,8 @@ public class WatermarkTimeTriggerPolicy<T> implements TriggerPolicy<T> {
long windowEndTs = nextWindowEndTs;
LOG.debug("Window end ts {} Watermark ts {}", windowEndTs, watermarkTs);
while (windowEndTs <= watermarkTs) {
- evictionPolicy.setContext(windowEndTs);
+ long currentCount = windowManager.getEventCount(windowEndTs);
+ evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount));
if (handler.onTrigger()) {
windowEndTs += slidingIntervalMs;
} else {
http://git-wip-us.apache.org/repos/asf/storm/blob/3cbd26ec/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java b/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
index a545427..6645566 100644
--- a/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
@@ -187,7 +187,7 @@ public class WindowManagerTest {
windowManager.add(i, now - 1000);
}
// simulate the time trigger by setting the reference time and invoking onTrigger() manually
- evictionPolicy.setContext(now + 100);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 100));
windowManager.onTrigger();
// 100 events with past ts should expire
@@ -210,7 +210,7 @@ public class WindowManagerTest {
}
activationsEvents.addAll(newEvents);
// simulate the time trigger by setting the reference time and invoking onTrigger() manually
- evictionPolicy.setContext(now + 200);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 200));
windowManager.onTrigger();
assertTrue(listener.onExpiryEvents.isEmpty());
assertEquals(activationsEvents, listener.onActivationEvents);
@@ -236,20 +236,20 @@ public class WindowManagerTest {
windowManager.add(i);
}
// simulate the time trigger by setting the reference time and invoking onTrigger() manually
- evictionPolicy.setContext(now + 60);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 60));
windowManager.onTrigger();
assertEquals(seq(1, 10), listener.onActivationEvents);
assertTrue(listener.onActivationExpiredEvents.isEmpty());
listener.clear();
// wait so all events expire
- evictionPolicy.setContext(now + 120);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 120));
windowManager.onTrigger();
assertEquals(seq(1, 10), listener.onExpiryEvents);
assertTrue(listener.onActivationEvents.isEmpty());
listener.clear();
- evictionPolicy.setContext(now + 180);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 180));
windowManager.onTrigger();
assertTrue(listener.onActivationExpiredEvents.isEmpty());
assertTrue(listener.onActivationEvents.isEmpty());
@@ -354,7 +354,7 @@ public class WindowManagerTest {
@Test
public void testCountBasedWindowWithEventTs() throws Exception {
- EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(3, windowManager);
+ EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(3);
windowManager.setEvictionPolicy(evictionPolicy);
TriggerPolicy<Integer> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager);
triggerPolicy.start();
@@ -430,7 +430,64 @@ public class WindowManagerTest {
assertEquals(seq(9), listener.allOnActivationEvents.get(0));
assertEquals(seq(9, 12), listener.allOnActivationEvents.get(1));
}
+
+ @Test
+ public void testCountBasedTumblingWithSameEventTs() throws Exception {
+ EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(2);
+ windowManager.setEvictionPolicy(evictionPolicy);
+ TriggerPolicy<Integer> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager);
+ triggerPolicy.start();
+ windowManager.setTriggerPolicy(triggerPolicy);
+
+ windowManager.add(1, 10);
+ windowManager.add(2, 10);
+ windowManager.add(3, 11);
+ windowManager.add(4, 12);
+ windowManager.add(5, 12);
+ windowManager.add(6, 12);
+ windowManager.add(7, 12);
+ windowManager.add(8, 13);
+ windowManager.add(9, 14);
+ windowManager.add(10, 15);
+
+ windowManager.add(new WaterMarkEvent<Integer>(20));
+ assertEquals(5, listener.allOnActivationEvents.size());
+ assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0));
+ assertEquals(seq(3, 4), listener.allOnActivationEvents.get(1));
+ assertEquals(seq(5, 6), listener.allOnActivationEvents.get(2));
+ assertEquals(seq(7, 8), listener.allOnActivationEvents.get(3));
+ assertEquals(seq(9, 10), listener.allOnActivationEvents.get(4));
+ }
+
@Test
+ public void testCountBasedSlidingWithSameEventTs() throws Exception {
+ EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(5);
+ windowManager.setEvictionPolicy(evictionPolicy);
+ TriggerPolicy<Integer> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager);
+ triggerPolicy.start();
+ windowManager.setTriggerPolicy(triggerPolicy);
+
+ windowManager.add(1, 10);
+ windowManager.add(2, 10);
+ windowManager.add(3, 11);
+ windowManager.add(4, 12);
+ windowManager.add(5, 12);
+ windowManager.add(6, 12);
+ windowManager.add(7, 12);
+ windowManager.add(8, 13);
+ windowManager.add(9, 14);
+ windowManager.add(10, 15);
+
+ windowManager.add(new WaterMarkEvent<Integer>(20));
+ assertEquals(5, listener.allOnActivationEvents.size());
+ assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0));
+ assertEquals(seq(1, 4), listener.allOnActivationEvents.get(1));
+ assertEquals(seq(2, 6), listener.allOnActivationEvents.get(2));
+ assertEquals(seq(4, 8), listener.allOnActivationEvents.get(3));
+ assertEquals(seq(6, 10), listener.allOnActivationEvents.get(4));
+
+ }
+ @Test
public void testEventTimeLag() throws Exception {
EvictionPolicy<Integer> evictionPolicy = new WatermarkTimeEvictionPolicy<>(20, 5);
windowManager.setEvictionPolicy(evictionPolicy);
[10/13] storm git commit: Unexpected behavior when using count window
together with timestamp extraction
Posted by pt...@apache.org.
Unexpected behavior when using count window together with timestamp extraction
Do not use timestamp to determine the event count in WatermarkCountEvictionPolicy
when count based trigger is used.
Signed-off-by: P. Taylor Goetz <pt...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f1eacf1c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f1eacf1c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f1eacf1c
Branch: refs/heads/1.x-branch
Commit: f1eacf1cef3af63ab8071026145cb6bbc6fed2c6
Parents: 88e8f17
Author: Arun Mahadevan <ar...@apache.org>
Authored: Thu Jul 14 00:00:57 2016 +0530
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 13:24:35 2016 -0400
----------------------------------------------------------------------
.../storm/topology/WindowedBoltExecutor.java | 2 +-
.../storm/windowing/CountEvictionPolicy.java | 12 ++--
.../storm/windowing/CountTriggerPolicy.java | 2 +-
.../storm/windowing/DefaultEvictionContext.java | 53 +++++++++++++++
.../apache/storm/windowing/EvictionContext.java | 47 +++++++++++++
.../apache/storm/windowing/EvictionPolicy.java | 3 +-
.../storm/windowing/TimeEvictionPolicy.java | 4 +-
.../storm/windowing/TimeTriggerPolicy.java | 2 +-
.../windowing/WatermarkCountEvictionPolicy.java | 27 +++++---
.../windowing/WatermarkCountTriggerPolicy.java | 2 +-
.../windowing/WatermarkTimeTriggerPolicy.java | 3 +-
.../storm/windowing/WindowManagerTest.java | 69 ++++++++++++++++++--
12 files changed, 197 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
index fe0f4da..3f245e1 100644
--- a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
+++ b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
@@ -253,7 +253,7 @@ public class WindowedBoltExecutor implements IRichBolt {
WindowManager<Tuple> manager) {
if (windowLengthCount != null) {
if (isTupleTs()) {
- return new WatermarkCountEvictionPolicy<>(windowLengthCount.value, manager);
+ return new WatermarkCountEvictionPolicy<>(windowLengthCount.value);
} else {
return new CountEvictionPolicy<>(windowLengthCount.value);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
index 0e7b233..fb12202 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
@@ -17,7 +17,7 @@
*/
package org.apache.storm.windowing;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
/**
* An eviction policy that tracks event counts and can
@@ -26,12 +26,12 @@ import java.util.concurrent.atomic.AtomicInteger;
* @param <T> the type of event tracked by this policy.
*/
public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
- private final int threshold;
- protected final AtomicInteger currentCount;
+ protected final int threshold;
+ protected final AtomicLong currentCount;
public CountEvictionPolicy(int count) {
this.threshold = count;
- this.currentCount = new AtomicInteger();
+ this.currentCount = new AtomicLong();
}
@Override
@@ -41,7 +41,7 @@ public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
* return if the event should be evicted
*/
while (true) {
- int curVal = currentCount.get();
+ long curVal = currentCount.get();
if (curVal > threshold) {
if (currentCount.compareAndSet(curVal, curVal - 1)) {
return Action.EXPIRE;
@@ -61,7 +61,7 @@ public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
}
@Override
- public void setContext(Object context) {
+ public void setContext(EvictionContext context) {
// NOOP
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
index 2594c67..17750b6 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
@@ -44,7 +44,7 @@ public class CountTriggerPolicy<T> implements TriggerPolicy<T> {
public void track(Event<T> event) {
if (started && !event.isWatermark()) {
if (currentCount.incrementAndGet() >= count) {
- evictionPolicy.setContext(System.currentTimeMillis());
+ evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis()));
handler.onTrigger();
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
new file mode 100644
index 0000000..ef65f66
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.windowing;
+
+public class DefaultEvictionContext implements EvictionContext {
+ private final Long referenceTime;
+ private final Long currentCount;
+ private final Long slidingCount;
+
+ public DefaultEvictionContext(Long referenceTime) {
+ this(referenceTime, null);
+ }
+
+ public DefaultEvictionContext(Long referenceTime, Long currentCount) {
+ this(referenceTime, currentCount, null);
+ }
+
+ public DefaultEvictionContext(Long referenceTime, Long currentCount, Long slidingCount) {
+ this.referenceTime = referenceTime;
+ this.currentCount = currentCount;
+ this.slidingCount = slidingCount;
+ }
+
+ @Override
+ public Long getReferenceTime() {
+ return referenceTime;
+ }
+
+ @Override
+ public Long getCurrentCount() {
+ return currentCount;
+ }
+
+ @Override
+ public Long getSlidingCount() {
+ return slidingCount;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
new file mode 100644
index 0000000..37dcfd9
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.windowing;
+
+/**
+ * Context information that can be used by the eviction policy
+ */
+public interface EvictionContext {
+ /**
+ * Returns the reference time that the eviction policy could use to
+ * evict the events. In the case of event time processing, this would be
+ * the watermark time.
+ *
+ * @return the reference time in millis
+ */
+ Long getReferenceTime();
+
+ /**
+ * Returns the sliding count for count based windows
+ *
+ * @return the sliding count
+ */
+ Long getSlidingCount();
+
+ /**
+ * Returns the current count of events in the queue up to the reference tim
+ * based on which count based evictions can be performed.
+ *
+ * @return the current count
+ */
+ Long getCurrentCount();
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
index 8128d80..05e4d93 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
@@ -70,5 +70,6 @@ public interface EvictionPolicy<T> {
*
* @param context
*/
- void setContext(Object context);
+ void setContext(EvictionContext context);
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
index a2db239..e646207 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
@@ -57,8 +57,8 @@ public class TimeEvictionPolicy<T> implements EvictionPolicy<T> {
}
@Override
- public void setContext(Object context) {
- referenceTime = ((Number) context).longValue();
+ public void setContext(EvictionContext context) {
+ referenceTime = context.getReferenceTime();
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
index 1ad7714..b057afb 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
@@ -115,7 +115,7 @@ public class TimeTriggerPolicy<T> implements TriggerPolicy<T> {
* to evict the events
*/
if (evictionPolicy != null) {
- evictionPolicy.setContext(System.currentTimeMillis());
+ evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis()));
}
handler.onTrigger();
} catch (Throwable th) {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
index e1af6fb..74240bb 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
@@ -24,25 +24,29 @@ package org.apache.storm.windowing;
* @param <T> the type of event tracked by this policy.
*/
public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
- private final WindowManager<T> windowManager;
/*
* The reference time in millis for window calculations and
* expiring events. If not set it will default to System.currentTimeMillis()
*/
private long referenceTime;
+ private long processed = 0L;
- public WatermarkCountEvictionPolicy(int count, WindowManager<T> windowManager) {
+ public WatermarkCountEvictionPolicy(int count) {
super(count);
- this.windowManager = windowManager;
}
@Override
public Action evict(Event<T> event) {
- if (event.getTimestamp() <= referenceTime) {
- return super.evict(event);
+ Action action;
+ if (event.getTimestamp() <= referenceTime && processed < currentCount.get()) {
+ action = super.evict(event);
+ if (action == Action.PROCESS) {
+ ++processed;
+ }
} else {
- return Action.KEEP;
+ action = Action.KEEP;
}
+ return action;
}
@Override
@@ -51,9 +55,14 @@ public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
}
@Override
- public void setContext(Object context) {
- referenceTime = (Long) context;
- currentCount.set(windowManager.getEventCount(referenceTime));
+ public void setContext(EvictionContext context) {
+ referenceTime = context.getReferenceTime();
+ if (context.getCurrentCount() != null) {
+ currentCount.set(context.getCurrentCount());
+ } else {
+ currentCount.set(processed + context.getSlidingCount());
+ }
+ processed = 0;
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
index 391efee..3cfcaad 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
@@ -74,7 +74,7 @@ public class WatermarkCountTriggerPolicy<T> implements TriggerPolicy<T> {
long watermarkTs = waterMarkEvent.getTimestamp();
List<Long> eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, count);
for (long ts : eventTs) {
- evictionPolicy.setContext(ts);
+ evictionPolicy.setContext(new DefaultEvictionContext(ts, null, Long.valueOf(count)));
handler.onTrigger();
lastProcessedTs = ts;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
index 3734c93..00620c4 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
@@ -75,7 +75,8 @@ public class WatermarkTimeTriggerPolicy<T> implements TriggerPolicy<T> {
long windowEndTs = nextWindowEndTs;
LOG.debug("Window end ts {} Watermark ts {}", windowEndTs, watermarkTs);
while (windowEndTs <= watermarkTs) {
- evictionPolicy.setContext(windowEndTs);
+ long currentCount = windowManager.getEventCount(windowEndTs);
+ evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount));
if (handler.onTrigger()) {
windowEndTs += slidingIntervalMs;
} else {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1eacf1c/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java b/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
index a545427..6645566 100644
--- a/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
@@ -187,7 +187,7 @@ public class WindowManagerTest {
windowManager.add(i, now - 1000);
}
// simulate the time trigger by setting the reference time and invoking onTrigger() manually
- evictionPolicy.setContext(now + 100);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 100));
windowManager.onTrigger();
// 100 events with past ts should expire
@@ -210,7 +210,7 @@ public class WindowManagerTest {
}
activationsEvents.addAll(newEvents);
// simulate the time trigger by setting the reference time and invoking onTrigger() manually
- evictionPolicy.setContext(now + 200);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 200));
windowManager.onTrigger();
assertTrue(listener.onExpiryEvents.isEmpty());
assertEquals(activationsEvents, listener.onActivationEvents);
@@ -236,20 +236,20 @@ public class WindowManagerTest {
windowManager.add(i);
}
// simulate the time trigger by setting the reference time and invoking onTrigger() manually
- evictionPolicy.setContext(now + 60);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 60));
windowManager.onTrigger();
assertEquals(seq(1, 10), listener.onActivationEvents);
assertTrue(listener.onActivationExpiredEvents.isEmpty());
listener.clear();
// wait so all events expire
- evictionPolicy.setContext(now + 120);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 120));
windowManager.onTrigger();
assertEquals(seq(1, 10), listener.onExpiryEvents);
assertTrue(listener.onActivationEvents.isEmpty());
listener.clear();
- evictionPolicy.setContext(now + 180);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 180));
windowManager.onTrigger();
assertTrue(listener.onActivationExpiredEvents.isEmpty());
assertTrue(listener.onActivationEvents.isEmpty());
@@ -354,7 +354,7 @@ public class WindowManagerTest {
@Test
public void testCountBasedWindowWithEventTs() throws Exception {
- EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(3, windowManager);
+ EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(3);
windowManager.setEvictionPolicy(evictionPolicy);
TriggerPolicy<Integer> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager);
triggerPolicy.start();
@@ -430,7 +430,64 @@ public class WindowManagerTest {
assertEquals(seq(9), listener.allOnActivationEvents.get(0));
assertEquals(seq(9, 12), listener.allOnActivationEvents.get(1));
}
+
+ @Test
+ public void testCountBasedTumblingWithSameEventTs() throws Exception {
+ EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(2);
+ windowManager.setEvictionPolicy(evictionPolicy);
+ TriggerPolicy<Integer> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager);
+ triggerPolicy.start();
+ windowManager.setTriggerPolicy(triggerPolicy);
+
+ windowManager.add(1, 10);
+ windowManager.add(2, 10);
+ windowManager.add(3, 11);
+ windowManager.add(4, 12);
+ windowManager.add(5, 12);
+ windowManager.add(6, 12);
+ windowManager.add(7, 12);
+ windowManager.add(8, 13);
+ windowManager.add(9, 14);
+ windowManager.add(10, 15);
+
+ windowManager.add(new WaterMarkEvent<Integer>(20));
+ assertEquals(5, listener.allOnActivationEvents.size());
+ assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0));
+ assertEquals(seq(3, 4), listener.allOnActivationEvents.get(1));
+ assertEquals(seq(5, 6), listener.allOnActivationEvents.get(2));
+ assertEquals(seq(7, 8), listener.allOnActivationEvents.get(3));
+ assertEquals(seq(9, 10), listener.allOnActivationEvents.get(4));
+ }
+
@Test
+ public void testCountBasedSlidingWithSameEventTs() throws Exception {
+ EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(5);
+ windowManager.setEvictionPolicy(evictionPolicy);
+ TriggerPolicy<Integer> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager);
+ triggerPolicy.start();
+ windowManager.setTriggerPolicy(triggerPolicy);
+
+ windowManager.add(1, 10);
+ windowManager.add(2, 10);
+ windowManager.add(3, 11);
+ windowManager.add(4, 12);
+ windowManager.add(5, 12);
+ windowManager.add(6, 12);
+ windowManager.add(7, 12);
+ windowManager.add(8, 13);
+ windowManager.add(9, 14);
+ windowManager.add(10, 15);
+
+ windowManager.add(new WaterMarkEvent<Integer>(20));
+ assertEquals(5, listener.allOnActivationEvents.size());
+ assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0));
+ assertEquals(seq(1, 4), listener.allOnActivationEvents.get(1));
+ assertEquals(seq(2, 6), listener.allOnActivationEvents.get(2));
+ assertEquals(seq(4, 8), listener.allOnActivationEvents.get(3));
+ assertEquals(seq(6, 10), listener.allOnActivationEvents.get(4));
+
+ }
+ @Test
public void testEventTimeLag() throws Exception {
EvictionPolicy<Integer> evictionPolicy = new WatermarkTimeEvictionPolicy<>(20, 5);
windowManager.setEvictionPolicy(evictionPolicy);
[02/13] storm git commit: STORM-1890: add maven depency to for
ring-core 1.3.1,
move org.apache.commons.FileUtils to 1.3.2 as that is needed by ring-core
1.3.1, move compojure to 1.1.9 since it matches 1.3.1
Posted by pt...@apache.org.
STORM-1890: add maven depency to for ring-core 1.3.1, move org.apache.commons.FileUtils to 1.3.2 as that is needed by ring-core 1.3.1, move compojure to 1.1.9 since it matches 1.3.1
Signed-off-by: P. Taylor Goetz <pt...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/51f0daa9
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/51f0daa9
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/51f0daa9
Branch: refs/heads/1.0.x-branch
Commit: 51f0daa9df14f499d83fdd513bbaed1d4f4fcf7b
Parents: 44e8170
Author: Alessandro Bellina <ab...@yahoo-inc.com>
Authored: Thu Jun 16 11:07:48 2016 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 12:14:18 2016 -0400
----------------------------------------------------------------------
pom.xml | 11 ++++++++---
storm-core/pom.xml | 4 ++++
2 files changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/51f0daa9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9062ada..2ea2184 100644
--- a/pom.xml
+++ b/pom.xml
@@ -200,19 +200,19 @@
<!-- dependency versions -->
<clojure.version>1.7.0</clojure.version>
<java_jmx.version>0.3.1</java_jmx.version>
- <compojure.version>1.1.3</compojure.version>
+ <compojure.version>1.1.9</compojure.version>
<hiccup.version>0.3.6</hiccup.version>
<commons-compress.version>1.4.1</commons-compress.version>
<commons-io.version>2.5</commons-io.version>
<commons-lang.version>2.5</commons-lang.version>
<commons-exec.version>1.1</commons-exec.version>
- <commons-fileupload.version>1.2.1</commons-fileupload.version>
+ <commons-fileupload.version>1.3.2</commons-fileupload.version>
<commons-codec.version>1.6</commons-codec.version>
<commons-cli.version>1.3.1</commons-cli.version>
<clj-time.version>0.8.0</clj-time.version>
<curator.version>2.10.0</curator.version>
<json-simple.version>1.1</json-simple.version>
- <ring.version>1.3.0</ring.version>
+ <ring.version>1.3.1</ring.version>
<ring-json.version>0.3.1</ring-json.version>
<jetty.version>7.6.13.v20130916</jetty.version>
<clojure.tools.logging.version>0.2.3</clojure.tools.logging.version>
@@ -659,6 +659,11 @@
</dependency>
<dependency>
<groupId>ring</groupId>
+ <artifactId>ring-core</artifactId>
+ <version>${ring.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>ring</groupId>
<artifactId>ring-devel</artifactId>
<version>${ring.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/storm/blob/51f0daa9/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index b79e923..17a9545 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -70,6 +70,10 @@
</dependency>
<dependency>
<groupId>ring</groupId>
+ <artifactId>ring-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>ring</groupId>
<artifactId>ring-devel</artifactId>
</dependency>
<dependency>
[11/13] storm git commit: add STORM-1964 to changelog
Posted by pt...@apache.org.
add STORM-1964 to changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6deb60fb
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6deb60fb
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6deb60fb
Branch: refs/heads/1.x-branch
Commit: 6deb60fb488e72b2802e23c7dbf13c53370be116
Parents: f1eacf1
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jul 21 13:25:02 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 13:25:02 2016 -0400
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6deb60fb/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 34ac20d..0e8d6d8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 1.1.0
+ * STORM-1964: Unexpected behavior when using count window together with timestamp extraction
* STORM-1890: ensure we refetch static resources after package build
* STORM-1988: Kafka Offset not showing due to bad classpath.
* STORM-1987: Fix TridentKafkaWordCount arg handling in distributed mode
[03/13] storm git commit: STORM-1890: add temp-file store since
wrap-multipart-params for ring 1.3 needs it
Posted by pt...@apache.org.
STORM-1890: add temp-file store since wrap-multipart-params for ring 1.3 needs it
Signed-off-by: P. Taylor Goetz <pt...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b52172b7
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b52172b7
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b52172b7
Branch: refs/heads/1.0.x-branch
Commit: b52172b7b4a8d41510fe96d918fd242b8ba7f4cc
Parents: 28ef601
Author: Alessandro Bellina <ab...@yahoo-inc.com>
Authored: Sat Jun 18 14:04:36 2016 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 12:14:18 2016 -0400
----------------------------------------------------------------------
storm-core/src/clj/org/apache/storm/ui/core.clj | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b52172b7/storm-core/src/clj/org/apache/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/ui/core.clj b/storm-core/src/clj/org/apache/storm/ui/core.clj
index 1ab2d33..30c5fc9 100644
--- a/storm-core/src/clj/org/apache/storm/ui/core.clj
+++ b/storm-core/src/clj/org/apache/storm/ui/core.clj
@@ -18,7 +18,8 @@
(:use compojure.core)
(:use [clojure.java.shell :only [sh]])
(:use ring.middleware.reload
- ring.middleware.multipart-params)
+ ring.middleware.multipart-params
+ ring.middleware.multipart-params.temp-file)
(:use [ring.middleware.json :only [wrap-json-params]])
(:use [hiccup core page-helpers])
(:use [org.apache.storm config util log stats zookeeper converter])
[06/13] storm git commit: add STORM-1890 to changelog
Posted by pt...@apache.org.
add STORM-1890 to changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b068dde1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b068dde1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b068dde1
Branch: refs/heads/1.0.x-branch
Commit: b068dde1e5e21da5f6088398ba01833e8f0411e9
Parents: 51f0daa
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jul 21 12:23:27 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 12:23:27 2016 -0400
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b068dde1/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 170aa4e..4182f1a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 1.0.2
+ * STORM-1890 ensure we refetch static resources after package build
* STORM-1939: Frequent InterruptedException raised by ShellBoltMessageQueue.poll
* STORM-1968: Storm logviewer does not work for nimbus.log in secure cluster
* STORM-1928: ShellSpout should check heartbeat while ShellSpout is waiting for subprocess to sync
[04/13] storm git commit: STORM-1890: add code Cache-Control:
no-cache to root html pages (/*.html)
Posted by pt...@apache.org.
STORM-1890: add code Cache-Control: no-cache to root html pages (/*.html)
Signed-off-by: P. Taylor Goetz <pt...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/44e81709
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/44e81709
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/44e81709
Branch: refs/heads/1.0.x-branch
Commit: 44e817092241613f5849ec6e8b6a84a034a2bb0e
Parents: b52172b
Author: Alessandro Bellina <ab...@yahoo-inc.com>
Authored: Sat Jun 18 14:05:27 2016 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 12:14:18 2016 -0400
----------------------------------------------------------------------
.../src/clj/org/apache/storm/ui/helpers.clj | 22 ++++++++++++++++----
1 file changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/44e81709/storm-core/src/clj/org/apache/storm/ui/helpers.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/ui/helpers.clj b/storm-core/src/clj/org/apache/storm/ui/helpers.clj
index 7ded154..81ceb58 100644
--- a/storm-core/src/clj/org/apache/storm/ui/helpers.clj
+++ b/storm-core/src/clj/org/apache/storm/ui/helpers.clj
@@ -32,18 +32,32 @@
[org.eclipse.jetty.util.ssl SslContextFactory]
[org.eclipse.jetty.server DispatcherType]
[org.eclipse.jetty.servlets CrossOriginFilter])
- (:require [ring.util servlet])
+ (:require [ring.util servlet]
+ [ring.util.response :as response])
(:require [compojure.route :as route]
[compojure.handler :as handler])
(:require [metrics.meters :refer [defmeter mark!]]))
(defmeter num-web-requests)
+
(defn requests-middleware
- "Coda Hale metric for counting the number of web requests."
+ "Wrap request with Coda Hale metric for counting the number of web requests,
+ and add Cache-Control: no-cache for html files in root directory (index.html, topology.html, etc)"
[handler]
(fn [req]
- (mark! num-web-requests)
- (handler req)))
+ (.mark num-web-requests)
+ (let [uri (:uri req)
+ res (handler req)
+ content-type (response/get-header res "Content-Type")]
+ ;; check that the response is html and that the path is for a root page: a single / in the path
+ ;; then we know we don't want it cached (e.g. /index.html)
+ (if (and (= content-type "text/html")
+ (= 1 (count (re-seq #"/" uri))))
+ ;; response for html page in root directory, no-cache
+ (response/header res "Cache-Control" "no-cache")
+ ;; else, carry on
+ res))))
+
(defn split-divide [val divider]
[(Integer. (int (/ val divider))) (mod val divider)]
[13/13] storm git commit: add STORM-1964 to changelog
Posted by pt...@apache.org.
add STORM-1964 to changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/bb1ea75e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/bb1ea75e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/bb1ea75e
Branch: refs/heads/1.0.x-branch
Commit: bb1ea75e2dab5802c844d6105332c3d4fb564397
Parents: 6c81357
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jul 21 13:26:26 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 13:26:26 2016 -0400
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/bb1ea75e/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0b7173a..1f15584 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 1.0.2
+ * STORM-1964: Unexpected behavior when using count window together with timestamp extraction
* STORM-1977: Restore logic: give up leadership when elected as leader but doesn't have one or more topology codes on local
* STORM-1890 ensure we refetch static resources after package build
* STORM-1730: LocalCluster#shutdown() does not terminate all storm threds/thread pools.
[09/13] storm git commit: add STORM-1964 to changelog
Posted by pt...@apache.org.
add STORM-1964 to changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5e5bb8f1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5e5bb8f1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5e5bb8f1
Branch: refs/heads/master
Commit: 5e5bb8f1ffff56026791563edede427b1ae6d9b1
Parents: a97b162
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jul 21 13:23:00 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 13:23:00 2016 -0400
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/5e5bb8f1/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f00febd..d660ebc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 2.0.0
+ * STORM-1964: Unexpected behavior when using count window together with timestamp extraction
* STORM-1890: ensure we refetch static resources after package build
* STORM-1966 Expand metric having Map type as value into multiple metrics based on entries
* STORM-1737: storm-kafka-client has compilation errors with Apache Kafka 0.10
[12/13] storm git commit: Unexpected behavior when using count window
together with timestamp extraction
Posted by pt...@apache.org.
Unexpected behavior when using count window together with timestamp extraction
Do not use timestamp to determine the event count in WatermarkCountEvictionPolicy
when count based trigger is used.
Signed-off-by: P. Taylor Goetz <pt...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6c813578
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6c813578
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6c813578
Branch: refs/heads/1.0.x-branch
Commit: 6c813578c07fa856ad930dbd02d192a752f837a0
Parents: b56a34a
Author: Arun Mahadevan <ar...@apache.org>
Authored: Thu Jul 14 00:00:57 2016 +0530
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 13:26:03 2016 -0400
----------------------------------------------------------------------
.../storm/topology/WindowedBoltExecutor.java | 2 +-
.../storm/windowing/CountEvictionPolicy.java | 12 ++--
.../storm/windowing/CountTriggerPolicy.java | 2 +-
.../storm/windowing/DefaultEvictionContext.java | 53 +++++++++++++++
.../apache/storm/windowing/EvictionContext.java | 47 +++++++++++++
.../apache/storm/windowing/EvictionPolicy.java | 3 +-
.../storm/windowing/TimeEvictionPolicy.java | 4 +-
.../storm/windowing/TimeTriggerPolicy.java | 2 +-
.../windowing/WatermarkCountEvictionPolicy.java | 27 +++++---
.../windowing/WatermarkCountTriggerPolicy.java | 2 +-
.../windowing/WatermarkTimeTriggerPolicy.java | 3 +-
.../storm/windowing/WindowManagerTest.java | 69 ++++++++++++++++++--
12 files changed, 197 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
index 9439ef2..0b18646 100644
--- a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
+++ b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
@@ -238,7 +238,7 @@ public class WindowedBoltExecutor implements IRichBolt {
WindowManager<Tuple> manager) {
if (windowLengthCount != null) {
if (isTupleTs()) {
- return new WatermarkCountEvictionPolicy<>(windowLengthCount.value, manager);
+ return new WatermarkCountEvictionPolicy<>(windowLengthCount.value);
} else {
return new CountEvictionPolicy<>(windowLengthCount.value);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
index 0e7b233..fb12202 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java
@@ -17,7 +17,7 @@
*/
package org.apache.storm.windowing;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
/**
* An eviction policy that tracks event counts and can
@@ -26,12 +26,12 @@ import java.util.concurrent.atomic.AtomicInteger;
* @param <T> the type of event tracked by this policy.
*/
public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
- private final int threshold;
- protected final AtomicInteger currentCount;
+ protected final int threshold;
+ protected final AtomicLong currentCount;
public CountEvictionPolicy(int count) {
this.threshold = count;
- this.currentCount = new AtomicInteger();
+ this.currentCount = new AtomicLong();
}
@Override
@@ -41,7 +41,7 @@ public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
* return if the event should be evicted
*/
while (true) {
- int curVal = currentCount.get();
+ long curVal = currentCount.get();
if (curVal > threshold) {
if (currentCount.compareAndSet(curVal, curVal - 1)) {
return Action.EXPIRE;
@@ -61,7 +61,7 @@ public class CountEvictionPolicy<T> implements EvictionPolicy<T> {
}
@Override
- public void setContext(Object context) {
+ public void setContext(EvictionContext context) {
// NOOP
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
index 2594c67..17750b6 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java
@@ -44,7 +44,7 @@ public class CountTriggerPolicy<T> implements TriggerPolicy<T> {
public void track(Event<T> event) {
if (started && !event.isWatermark()) {
if (currentCount.incrementAndGet() >= count) {
- evictionPolicy.setContext(System.currentTimeMillis());
+ evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis()));
handler.onTrigger();
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
new file mode 100644
index 0000000..ef65f66
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.windowing;
+
+public class DefaultEvictionContext implements EvictionContext {
+ private final Long referenceTime;
+ private final Long currentCount;
+ private final Long slidingCount;
+
+ public DefaultEvictionContext(Long referenceTime) {
+ this(referenceTime, null);
+ }
+
+ public DefaultEvictionContext(Long referenceTime, Long currentCount) {
+ this(referenceTime, currentCount, null);
+ }
+
+ public DefaultEvictionContext(Long referenceTime, Long currentCount, Long slidingCount) {
+ this.referenceTime = referenceTime;
+ this.currentCount = currentCount;
+ this.slidingCount = slidingCount;
+ }
+
+ @Override
+ public Long getReferenceTime() {
+ return referenceTime;
+ }
+
+ @Override
+ public Long getCurrentCount() {
+ return currentCount;
+ }
+
+ @Override
+ public Long getSlidingCount() {
+ return slidingCount;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
new file mode 100644
index 0000000..37dcfd9
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.windowing;
+
+/**
+ * Context information that can be used by the eviction policy
+ */
+public interface EvictionContext {
+ /**
+ * Returns the reference time that the eviction policy could use to
+ * evict the events. In the case of event time processing, this would be
+ * the watermark time.
+ *
+ * @return the reference time in millis
+ */
+ Long getReferenceTime();
+
+ /**
+ * Returns the sliding count for count based windows
+ *
+ * @return the sliding count
+ */
+ Long getSlidingCount();
+
+ /**
+ * Returns the current count of events in the queue up to the reference tim
+ * based on which count based evictions can be performed.
+ *
+ * @return the current count
+ */
+ Long getCurrentCount();
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
index 8128d80..05e4d93 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
@@ -70,5 +70,6 @@ public interface EvictionPolicy<T> {
*
* @param context
*/
- void setContext(Object context);
+ void setContext(EvictionContext context);
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
index a2db239..e646207 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
@@ -57,8 +57,8 @@ public class TimeEvictionPolicy<T> implements EvictionPolicy<T> {
}
@Override
- public void setContext(Object context) {
- referenceTime = ((Number) context).longValue();
+ public void setContext(EvictionContext context) {
+ referenceTime = context.getReferenceTime();
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
index 1ad7714..b057afb 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java
@@ -115,7 +115,7 @@ public class TimeTriggerPolicy<T> implements TriggerPolicy<T> {
* to evict the events
*/
if (evictionPolicy != null) {
- evictionPolicy.setContext(System.currentTimeMillis());
+ evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis()));
}
handler.onTrigger();
} catch (Throwable th) {
http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
index e1af6fb..74240bb 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
@@ -24,25 +24,29 @@ package org.apache.storm.windowing;
* @param <T> the type of event tracked by this policy.
*/
public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
- private final WindowManager<T> windowManager;
/*
* The reference time in millis for window calculations and
* expiring events. If not set it will default to System.currentTimeMillis()
*/
private long referenceTime;
+ private long processed = 0L;
- public WatermarkCountEvictionPolicy(int count, WindowManager<T> windowManager) {
+ public WatermarkCountEvictionPolicy(int count) {
super(count);
- this.windowManager = windowManager;
}
@Override
public Action evict(Event<T> event) {
- if (event.getTimestamp() <= referenceTime) {
- return super.evict(event);
+ Action action;
+ if (event.getTimestamp() <= referenceTime && processed < currentCount.get()) {
+ action = super.evict(event);
+ if (action == Action.PROCESS) {
+ ++processed;
+ }
} else {
- return Action.KEEP;
+ action = Action.KEEP;
}
+ return action;
}
@Override
@@ -51,9 +55,14 @@ public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
}
@Override
- public void setContext(Object context) {
- referenceTime = (Long) context;
- currentCount.set(windowManager.getEventCount(referenceTime));
+ public void setContext(EvictionContext context) {
+ referenceTime = context.getReferenceTime();
+ if (context.getCurrentCount() != null) {
+ currentCount.set(context.getCurrentCount());
+ } else {
+ currentCount.set(processed + context.getSlidingCount());
+ }
+ processed = 0;
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
index 391efee..3cfcaad 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java
@@ -74,7 +74,7 @@ public class WatermarkCountTriggerPolicy<T> implements TriggerPolicy<T> {
long watermarkTs = waterMarkEvent.getTimestamp();
List<Long> eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, count);
for (long ts : eventTs) {
- evictionPolicy.setContext(ts);
+ evictionPolicy.setContext(new DefaultEvictionContext(ts, null, Long.valueOf(count)));
handler.onTrigger();
lastProcessedTs = ts;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
index 3734c93..00620c4 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java
@@ -75,7 +75,8 @@ public class WatermarkTimeTriggerPolicy<T> implements TriggerPolicy<T> {
long windowEndTs = nextWindowEndTs;
LOG.debug("Window end ts {} Watermark ts {}", windowEndTs, watermarkTs);
while (windowEndTs <= watermarkTs) {
- evictionPolicy.setContext(windowEndTs);
+ long currentCount = windowManager.getEventCount(windowEndTs);
+ evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount));
if (handler.onTrigger()) {
windowEndTs += slidingIntervalMs;
} else {
http://git-wip-us.apache.org/repos/asf/storm/blob/6c813578/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java b/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
index a545427..6645566 100644
--- a/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
@@ -187,7 +187,7 @@ public class WindowManagerTest {
windowManager.add(i, now - 1000);
}
// simulate the time trigger by setting the reference time and invoking onTrigger() manually
- evictionPolicy.setContext(now + 100);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 100));
windowManager.onTrigger();
// 100 events with past ts should expire
@@ -210,7 +210,7 @@ public class WindowManagerTest {
}
activationsEvents.addAll(newEvents);
// simulate the time trigger by setting the reference time and invoking onTrigger() manually
- evictionPolicy.setContext(now + 200);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 200));
windowManager.onTrigger();
assertTrue(listener.onExpiryEvents.isEmpty());
assertEquals(activationsEvents, listener.onActivationEvents);
@@ -236,20 +236,20 @@ public class WindowManagerTest {
windowManager.add(i);
}
// simulate the time trigger by setting the reference time and invoking onTrigger() manually
- evictionPolicy.setContext(now + 60);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 60));
windowManager.onTrigger();
assertEquals(seq(1, 10), listener.onActivationEvents);
assertTrue(listener.onActivationExpiredEvents.isEmpty());
listener.clear();
// wait so all events expire
- evictionPolicy.setContext(now + 120);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 120));
windowManager.onTrigger();
assertEquals(seq(1, 10), listener.onExpiryEvents);
assertTrue(listener.onActivationEvents.isEmpty());
listener.clear();
- evictionPolicy.setContext(now + 180);
+ evictionPolicy.setContext(new DefaultEvictionContext(now + 180));
windowManager.onTrigger();
assertTrue(listener.onActivationExpiredEvents.isEmpty());
assertTrue(listener.onActivationEvents.isEmpty());
@@ -354,7 +354,7 @@ public class WindowManagerTest {
@Test
public void testCountBasedWindowWithEventTs() throws Exception {
- EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(3, windowManager);
+ EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(3);
windowManager.setEvictionPolicy(evictionPolicy);
TriggerPolicy<Integer> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager);
triggerPolicy.start();
@@ -430,7 +430,64 @@ public class WindowManagerTest {
assertEquals(seq(9), listener.allOnActivationEvents.get(0));
assertEquals(seq(9, 12), listener.allOnActivationEvents.get(1));
}
+
+ @Test
+ public void testCountBasedTumblingWithSameEventTs() throws Exception {
+ EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(2);
+ windowManager.setEvictionPolicy(evictionPolicy);
+ TriggerPolicy<Integer> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager);
+ triggerPolicy.start();
+ windowManager.setTriggerPolicy(triggerPolicy);
+
+ windowManager.add(1, 10);
+ windowManager.add(2, 10);
+ windowManager.add(3, 11);
+ windowManager.add(4, 12);
+ windowManager.add(5, 12);
+ windowManager.add(6, 12);
+ windowManager.add(7, 12);
+ windowManager.add(8, 13);
+ windowManager.add(9, 14);
+ windowManager.add(10, 15);
+
+ windowManager.add(new WaterMarkEvent<Integer>(20));
+ assertEquals(5, listener.allOnActivationEvents.size());
+ assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0));
+ assertEquals(seq(3, 4), listener.allOnActivationEvents.get(1));
+ assertEquals(seq(5, 6), listener.allOnActivationEvents.get(2));
+ assertEquals(seq(7, 8), listener.allOnActivationEvents.get(3));
+ assertEquals(seq(9, 10), listener.allOnActivationEvents.get(4));
+ }
+
@Test
+ public void testCountBasedSlidingWithSameEventTs() throws Exception {
+ EvictionPolicy<Integer> evictionPolicy = new WatermarkCountEvictionPolicy<>(5);
+ windowManager.setEvictionPolicy(evictionPolicy);
+ TriggerPolicy<Integer> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager);
+ triggerPolicy.start();
+ windowManager.setTriggerPolicy(triggerPolicy);
+
+ windowManager.add(1, 10);
+ windowManager.add(2, 10);
+ windowManager.add(3, 11);
+ windowManager.add(4, 12);
+ windowManager.add(5, 12);
+ windowManager.add(6, 12);
+ windowManager.add(7, 12);
+ windowManager.add(8, 13);
+ windowManager.add(9, 14);
+ windowManager.add(10, 15);
+
+ windowManager.add(new WaterMarkEvent<Integer>(20));
+ assertEquals(5, listener.allOnActivationEvents.size());
+ assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0));
+ assertEquals(seq(1, 4), listener.allOnActivationEvents.get(1));
+ assertEquals(seq(2, 6), listener.allOnActivationEvents.get(2));
+ assertEquals(seq(4, 8), listener.allOnActivationEvents.get(3));
+ assertEquals(seq(6, 10), listener.allOnActivationEvents.get(4));
+
+ }
+ @Test
public void testEventTimeLag() throws Exception {
EvictionPolicy<Integer> evictionPolicy = new WatermarkTimeEvictionPolicy<>(20, 5);
windowManager.setEvictionPolicy(evictionPolicy);
[05/13] storm git commit: STORM-1890: ensure static UI resources are
re-requested with different Storm package builds
Posted by pt...@apache.org.
STORM-1890: ensure static UI resources are re-requested with different Storm package builds
Signed-off-by: P. Taylor Goetz <pt...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/28ef601f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/28ef601f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/28ef601f
Branch: refs/heads/1.0.x-branch
Commit: 28ef601fca4721ec5b285f4aac55dddee783a6df
Parents: f81f30b
Author: Alessandro Bellina <ab...@yahoo-inc.com>
Authored: Tue Jun 14 07:39:45 2016 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 12:14:18 2016 -0400
----------------------------------------------------------------------
storm-core/src/ui/public/component.html | 30 ++++++++++----------
.../src/ui/public/deep_search_result.html | 10 +++----
storm-core/src/ui/public/index.html | 18 ++++++------
storm-core/src/ui/public/js/script.js | 12 ++++++++
storm-core/src/ui/public/js/visualization.js | 4 +--
storm-core/src/ui/public/logviewer_search.html | 6 ++--
storm-core/src/ui/public/search_result.html | 6 ++--
storm-core/src/ui/public/topology.html | 10 +++----
storm-dist/binary/pom.xml | 10 +++++++
storm-dist/binary/src/main/assembly/binary.xml | 25 +++++++++++++++-
10 files changed, 88 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/component.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/component.html b/storm-core/src/ui/public/component.html
index 37ee525..fc7b15e 100644
--- a/storm-core/src/ui/public/component.html
+++ b/storm-core/src/ui/public/component.html
@@ -22,7 +22,7 @@
<link href="/css/bootstrap-3.3.1.min.css" rel="stylesheet" type="text/css">
<link href="/css/jquery.dataTables.1.10.4.min.css" rel="stylesheet" type="text/css">
<link href="/css/dataTables.bootstrap.css" rel="stylesheet" type="text/css">
-<link href="/css/style.css" rel="stylesheet" type="text/css">
+<link href="/css/style.css?_ts=${packageTimestamp}" rel="stylesheet" type="text/css">
<script src="/js/jquery-1.11.1.min.js" type="text/javascript"></script>
<script src="/js/jquery.dataTables.1.10.4.min.js" type="text/javascript"></script>
<script src="/js/jquery.cookies.2.2.0.min.js" type="text/javascript"></script>
@@ -32,7 +32,7 @@
<script src="/js/jquery.blockUI.min.js" type="text/javascript"></script>
<script src="/js/moment.min.js" type="text/javascript"></script>
<script src="/js/dataTables.bootstrap.min.js" type="text/javascript"></script>
-<script src="/js/script.js" type="text/javascript"></script>
+<script src="/js/script.js?_ts=${packageTimestamp}" type="text/javascript"></script>
</head>
<body>
<div class="container-fluid">
@@ -83,7 +83,7 @@ function jsError(other) {
try {
other();
} catch (err) {
- $.get("/templates/json-error-template.html", function(template) {
+ getStatic("/templates/json-error-template.html", function(template) {
$("#json-response-error").append(Mustache.render($(template).filter("#json-error-template").html(),{error: "JS Error", errorMessage: err}));
});
}
@@ -156,7 +156,7 @@ $(document).ready(function() {
$.ajaxSetup({
"error":function(jqXHR,textStatus,response) {
var errorJson = jQuery.parseJSON(jqXHR.responseText);
- $.get("/templates/json-error-template.html", function(template) {
+ getStatic("/templates/json-error-template.html", function(template) {
$("#json-response-error").append(Mustache.render($(template).filter("#json-error-template").html(),errorJson));
});
}
@@ -187,7 +187,7 @@ $(document).ready(function() {
$.getJSON(url,function(response,status,jqXHR) {
var uiUser = $("#ui-user");
- $.get("/templates/user-template.html", function(template) {
+ getStatic("/templates/user-template.html", function(template) {
uiUser.append(Mustache.render($(template).filter("#user-template").html(),response));
$('#ui-user [data-toggle="tooltip"]').tooltip()
});
@@ -218,7 +218,7 @@ $(document).ready(function() {
var profilerControl = $("#profiler-control");
var executorStats = $("#component-executor-stats");
var componentErrors = $("#component-errors");
- $.get("/templates/component-page-template.html", function(template) {
+ getStatic("/templates/component-page-template.html", function(template) {
response["profilerActive"] = $.map(response["profilerActive"], function(active_map) {
var date = new Date();
var millis = date.getTime() + parseInt(active_map["timestamp"]);
@@ -357,9 +357,9 @@ function start_profiling() {
var passed = {}
Object.keys(workerActionSelected).forEach(function (id) {
var url = "/api/v1/topology/"+topologyId+"/profiling/start/" + id + "/" + timeout;
- $.get(url, function(response,status,jqXHR) {
+ getStatic(url, function(response,status,jqXHR) {
jsError(function() {
- $.get("/templates/component-page-template.html", function(template) {
+ getStatic("/templates/component-page-template.html", function(template) {
var host_port_split = id.split(":");
var host = host_port_split[0];
var port = host_port_split[1];
@@ -398,7 +398,7 @@ function stop_profiling(id) {
$("#stop_" + id).prop('disabled', true);
setTimeout(function(){ $("#stop_" + id).prop('disabled', false); }, 5000);
- $.get(url, function(response,status,jqXHR) {
+ getStatic(url, function(response,status,jqXHR) {
alert("Submitted request to stop profiling...");
})
.fail(function(response) {
@@ -414,7 +414,7 @@ function dump_profile(id) {
$("#dump_profile_" + id).prop('disabled', true);
setTimeout(function(){ $("#dump_profile_" + id).prop('disabled', false); }, 5000);
- $.get(url, function(response,status,jqXHR) {
+ getStatic(url, function(response,status,jqXHR) {
alert("Submitted request to dump profile snapshot...");
})
.fail(function(response) {
@@ -434,7 +434,7 @@ function dump_jstacks() {
$("#dump_jstack_" + id).prop('disabled', true);
setTimeout(function(){ $("#dump_jstack_" + id).prop('disabled', false); }, 5000);
- $.get(url)
+ getStatic(url)
.fail(function(response) {
failed[id] = response;
});
@@ -459,7 +459,7 @@ function dump_jstack(id) {
$("#dump_jstack_" + id).prop('disabled', true);
setTimeout(function(){ $("#dump_jstack_" + id).prop('disabled', false); }, 5000);
- $.get(url, function(response,status,jqXHR) {
+ getStatic(url, function(response,status,jqXHR) {
alert("Submitted request for jstack dump...");
})
.fail(function(response) {
@@ -477,7 +477,7 @@ function restart_worker_jvms() {
$("#restart_worker_jvm_" + id).prop('disabled', true);
setTimeout(function(){ $("#restart_worker_jvm_" + id).prop('disabled', false); }, 5000);
- $.get(url)
+ getStatic(url)
.fail(function(response) {
failed[id] = response;
});
@@ -505,7 +505,7 @@ function dump_heaps() {
$("#dump_heap_" + id).prop('disabled', true);
setTimeout(function(){ $("#dump_heap_" + id).prop('disabled', false); }, 5000);
- $.get(url)
+ getStatic(url)
.fail(function(response) {
failed[id] = response;
});
@@ -530,7 +530,7 @@ function dump_heap(id) {
$("#dump_heap_" + id).prop('disabled', true);
setTimeout(function(){ $("#dump_heap_" + id).prop('disabled', false); }, 5000);
- $.get(url, function(response,status,jqXHR) {
+ getStatic(url, function(response,status,jqXHR) {
alert("Submitted request for jmap dump...");
})
.fail(function(response) {
http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/deep_search_result.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/deep_search_result.html b/storm-core/src/ui/public/deep_search_result.html
index 406a101..5b4f47b 100644
--- a/storm-core/src/ui/public/deep_search_result.html
+++ b/storm-core/src/ui/public/deep_search_result.html
@@ -21,7 +21,7 @@
<link href="/css/bootstrap-3.3.1.min.css" rel="stylesheet" type="text/css">
<link href="/css/jquery.dataTables.1.10.4.min.css" rel="stylesheet" type="text/css">
<link href="/css/dataTables.bootstrap.css" rel="stylesheet" type="text/css">
-<link href="/css/style.css" rel="stylesheet" type="text/css">
+<link href="/css/style.css?_ts=${packageTimestamp}" rel="stylesheet" type="text/css">
<script src="/js/jquery-1.11.1.min.js" type="text/javascript"></script>
<script src="/js/jquery.dataTables.1.10.4.min.js" type="text/javascript"></script>
<script src="/js/jquery.cookies.2.2.0.min.js" type="text/javascript"></script>
@@ -31,7 +31,7 @@
<script src="/js/jquery.blockUI.min.js" type="text/javascript"></script>
<script src="/js/url.min.js" type="text/javascript"></script>
<script src="/js/dataTables.bootstrap.min.js" type="text/javascript"></script>
-<script src="/js/script.js" type="text/javascript"></script>
+<script src="/js/script.js?_ts=${packageTimestamp}" type="text/javascript"></script>
</head>
<body>
<div class="container-fluid">
@@ -56,12 +56,12 @@ $(document).ready(function() {
var port = $.url("?port") || "*";
var search_archived = $.url("?search-archived");
- $.get("/templates/deep-search-result-page-template.html", function(template) {
+ getStatic("/templates/deep-search-result-page-template.html", function(template) {
var checked;
if (search_archived) {
checked = "checked";
}
- $.get("/api/v1/history/summary", function (response, status, jqXHR) {
+ getStatic("/api/v1/history/summary", function (response, status, jqXHR) {
var findIds = function findIds(query, cb) {
var found = [];
var re = new RegExp(query, 'i');
@@ -104,7 +104,7 @@ $(document).ready(function() {
var result = $("#result");
if(search) {
- $.get("/api/v1/supervisor/summary", function(response, status, jqXHR) {
+ getStatic("/api/v1/supervisor/summary", function(response, status, jqXHR) {
for(var i in response.supervisors) {
response.supervisors[i].elemId = elem_id_for_host(response.supervisors[i].host);
http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/index.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/index.html b/storm-core/src/ui/public/index.html
index eae423c..52900a5 100644
--- a/storm-core/src/ui/public/index.html
+++ b/storm-core/src/ui/public/index.html
@@ -23,7 +23,7 @@
<link href="/css/jquery.dataTables.1.10.4.min.css" rel="stylesheet" type="text/css">
<link href="/css/dataTables.bootstrap.css" rel="stylesheet" type="text/css">
<link href="/css/jsonFormatter.min.css" rel="stylesheet" type="text/css">
-<link href="/css/style.css" rel="stylesheet" type="text/css">
+<link href="/css/style.css?_ts=${packageTimestamp}" rel="stylesheet" type="text/css">
<script src="/js/jquery-1.11.1.min.js" type="text/javascript"></script>
<script src="/js/jquery.dataTables.1.10.4.min.js" type="text/javascript"></script>
<script src="/js/jquery.cookies.2.2.0.min.js" type="text/javascript"></script>
@@ -32,7 +32,7 @@
<script src="/js/jquery.blockUI.min.js" type="text/javascript"></script>
<script src="/js/dataTables.bootstrap.min.js" type="text/javascript"></script>
<script src="/js/jsonFormatter.min.js" type="text/javascript"></script>
-<script src="/js/script.js" type="text/javascript"></script>
+<script src="/js/script.js?_ts=${packageTimestamp}" type="text/javascript"></script>
</head>
<body>
<div class="container-fluid">
@@ -96,7 +96,7 @@ $(document).ready(function() {
$.ajaxSetup({
"error":function(jqXHR,textStatus,response) {
var errorJson = jQuery.parseJSON(jqXHR.responseText);
- $.get("/templates/json-error-template.html", function(template) {
+ getStatic("/templates/json-error-template.html", function(template) {
$("#json-response-error").append(Mustache.render($(template).filter("#json-error-template").html(),errorJson));
});
}
@@ -109,18 +109,18 @@ $(document).ready(function() {
var config = $("#nimbus-configuration");
$.getJSON("/api/v1/cluster/summary",function(response,status,jqXHR) {
- $.get("/templates/user-template.html", function(template) {
+ getStatic("/templates/user-template.html", function(template) {
uiUser.append(Mustache.render($(template).filter("#user-template").html(),response));
$('#ui-user [data-toggle="tooltip"]').tooltip()
});
- $.get("/templates/index-page-template.html", function(template) {
+ getStatic("/templates/index-page-template.html", function(template) {
clusterSummary.append(Mustache.render($(template).filter("#cluster-summary-template").html(),response));
$('#cluster-summary [data-toggle="tooltip"]').tooltip();
});
});
$.getJSON("/api/v1/nimbus/summary",function(response,status,jqXHR) {
- $.get("/templates/index-page-template.html", function(template) {
+ getStatic("/templates/index-page-template.html", function(template) {
nimbusSummary.append(Mustache.render($(template).filter("#nimbus-summary-template").html(),response));
//host, port, isLeader, version, uptime
dtAutoPage("#nimbus-summary-table", {
@@ -133,7 +133,7 @@ $(document).ready(function() {
});
});
$.getJSON("/api/v1/topology/summary",function(response,status,jqXHR) {
- $.get("/templates/index-page-template.html", function(template) {
+ getStatic("/templates/index-page-template.html", function(template) {
topologySummary.append(Mustache.render($(template).filter("#topology-summary-template").html(),response));
//name, owner, status, uptime, num workers, num executors, num tasks, replication count, assigned total mem, assigned total cpu, scheduler info
dtAutoPage("#topology-summary-table", {
@@ -146,7 +146,7 @@ $(document).ready(function() {
});
});
$.getJSON("/api/v1/supervisor/summary",function(response,status,jqXHR) {
- $.get("/templates/index-page-template.html", function(template) {
+ getStatic("/templates/index-page-template.html", function(template) {
supervisorSummary.append(Mustache.render($(template).filter("#supervisor-summary-template").html(),response));
//id, host, uptime, slots, used slots
dtAutoPage("#supervisor-summary-table", {
@@ -160,7 +160,7 @@ $(document).ready(function() {
});
$.getJSON("/api/v1/cluster/configuration",function(response,status,jqXHR) {
var formattedResponse = formatConfigData(response);
- $.get("/templates/index-page-template.html", function(template) {
+ getStatic("/templates/index-page-template.html", function(template) {
config.append(Mustache.render($(template).filter("#configuration-template").html(),formattedResponse));
$('#nimbus-configuration-table td').jsonFormatter()
//key, value
http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/js/script.js
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/js/script.js b/storm-core/src/ui/public/js/script.js
index ff3c525..769442f 100644
--- a/storm-core/src/ui/public/js/script.js
+++ b/storm-core/src/ui/public/js/script.js
@@ -259,3 +259,15 @@ $.blockUI.defaults.css = {
opacity: .5,
color: '#fff',margin:0,width:"30%",top:"40%",left:"35%",textAlign:"center"
};
+
+// add a url query param to static (templates normally) ajax requests
+// for cache busting
+function getStatic(url, cb) {
+ return $.ajax({
+ url: url,
+ data: {
+ '_ts': '${packageTimestamp}'
+ },
+ success: cb
+ });
+};
http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/js/visualization.js
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/js/visualization.js b/storm-core/src/ui/public/js/visualization.js
index 9aab087..4daadc2 100644
--- a/storm-core/src/ui/public/js/visualization.js
+++ b/storm-core/src/ui/public/js/visualization.js
@@ -379,7 +379,7 @@ function jsError(other) {
try {
other();
} catch (err) {
- $.get("/templates/json-error-template.html", function(template) {
+ getStatic("/templates/json-error-template.html", function(template) {
$("#json-response-error").append(Mustache.render($(template).filter("#json-error-template").html(),{error: "JS Error", errorMessage: err}));
});
}
@@ -388,7 +388,7 @@ function jsError(other) {
var should_update;
function show_visualization(sys) {
$.getJSON("/api/v1/topology/"+$.url("?id")+"/visualization-init",function(response,status,jqXHR) {
- $.get("/templates/topology-page-template.html", function(template) {
+ getStatic("/templates/topology-page-template.html", function(template) {
jsError(function() {
var topologyVisualization = $("#visualization-container");
topologyVisualization.append(
http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/logviewer_search.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/logviewer_search.html b/storm-core/src/ui/public/logviewer_search.html
index f981499..3a1682e 100644
--- a/storm-core/src/ui/public/logviewer_search.html
+++ b/storm-core/src/ui/public/logviewer_search.html
@@ -21,7 +21,7 @@
<link href="/css/bootstrap-3.3.1.min.css" rel="stylesheet" type="text/css">
<link href="/css/jquery.dataTables.1.10.4.min.css" rel="stylesheet" type="text/css">
<link href="/css/dataTables.bootstrap.css" rel="stylesheet" type="text/css">
-<link href="/css/style.css" rel="stylesheet" type="text/css">
+<link href="/css/style.css?_ts=${packageTimestamp}" rel="stylesheet" type="text/css">
<script src="/js/jquery-1.11.1.min.js" type="text/javascript"></script>
<script src="/js/jquery.dataTables.1.10.4.min.js" type="text/javascript"></script>
<script src="/js/jquery.cookies.2.2.0.min.js" type="text/javascript"></script>
@@ -30,7 +30,7 @@
<script src="/js/jquery.blockUI.min.js" type="text/javascript"></script>
<script src="/js/url.min.js" type="text/javascript"></script>
<script src="/js/dataTables.bootstrap.min.js" type="text/javascript"></script>
-<script src="/js/script.js" type="text/javascript"></script>
+<script src="/js/script.js?_ts=${packageTimestamp}" type="text/javascript"></script>
</head>
<body>
<div class="container-fluid">
@@ -51,7 +51,7 @@ $(document).ready(function() {
file = decodeURIComponent(file);
search = decodeURIComponent(search);
- $.get("/templates/logviewer-search-page-template.html", function(template) {
+ getStatic("/templates/logviewer-search-page-template.html", function(template) {
$("#search-form").append(Mustache.render($(template).filter("#search-single-file").html(),{file: file, search: search, isDaemon: isDaemon}));
var result = $("#result");
http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/search_result.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/search_result.html b/storm-core/src/ui/public/search_result.html
index ec44aa9..ee0efd8 100644
--- a/storm-core/src/ui/public/search_result.html
+++ b/storm-core/src/ui/public/search_result.html
@@ -21,7 +21,7 @@
<link href="/css/bootstrap-3.3.1.min.css" rel="stylesheet" type="text/css">
<link href="/css/jquery.dataTables.1.10.4.min.css" rel="stylesheet" type="text/css">
<link href="/css/dataTables.bootstrap.css" rel="stylesheet" type="text/css">
-<link href="/css/style.css" rel="stylesheet" type="text/css">
+<link href="/css/style.css?_ts=${packageTimestamp}" rel="stylesheet" type="text/css">
<script src="/js/jquery-1.11.1.min.js" type="text/javascript"></script>
<script src="/js/jquery.dataTables.1.10.4.min.js" type="text/javascript"></script>
<script src="/js/jquery.cookies.2.2.0.min.js" type="text/javascript"></script>
@@ -30,7 +30,7 @@
<script src="/js/jquery.blockUI.min.js" type="text/javascript"></script>
<script src="/js/url.min.js" type="text/javascript"></script>
<script src="/js/dataTables.bootstrap.min.js" type="text/javascript"></script>
-<script src="/js/script.js" type="text/javascript"></script>
+<script src="/js/script.js?_ts=${packageTimestamp}" type="text/javascript"></script>
</head>
<body>
<div class="container-fluid">
@@ -49,7 +49,7 @@ $(document).ready(function() {
var count = $.url("?count") || 2;
var searchArchived = $.url("?searchArchived") || "";
- $.get("/templates/search-result-page-template.html", function(template) {
+ getStatic("/templates/search-result-page-template.html", function(template) {
$("#search-form").append(Mustache.render($(template).filter("#search-form-template").html(),{id: id, search: search, count: count, searchArchived: searchArchived}));
var result = $("#result");
http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-core/src/ui/public/topology.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/topology.html b/storm-core/src/ui/public/topology.html
index 061fd91..f955165 100644
--- a/storm-core/src/ui/public/topology.html
+++ b/storm-core/src/ui/public/topology.html
@@ -24,7 +24,7 @@
<link href="/css/jquery.dataTables.1.10.4.min.css" rel="stylesheet" type="text/css">
<link href="/css/dataTables.bootstrap.css" rel="stylesheet" type="text/css">
<link href="/css/jsonFormatter.min.css" rel="stylesheet" type="text/css">
-<link href="/css/style.css" rel="stylesheet" type="text/css">
+<link href="/css/style.css?_ts=${packageTimestamp}" rel="stylesheet" type="text/css">
<script src="/js/jquery-1.11.1.min.js" type="text/javascript"></script>
<script src="/js/jquery.dataTables.1.10.4.min.js" type="text/javascript"></script>
<script src="/js/jquery.cookies.2.2.0.min.js" type="text/javascript"></script>
@@ -33,7 +33,7 @@
<script src="/js/bootstrap-3.3.1.min.js" type="text/javascript"></script>
<script src="/js/jquery.blockUI.min.js" type="text/javascript"></script>
<script src="/js/jsonFormatter.min.js" type="text/javascript"></script>
-<script src="/js/script.js" type="text/javascript"></script>
+<script src="/js/script.js?_ts=${packageTimestamp}" type="text/javascript"></script>
<script src="/js/visualization.js" type="text/javascript"></script>
<script src="/js/arbor.js" type="text/javascript"></script>
<script src="/js/arbor-graphics.js" type="text/javascript"></script>
@@ -227,7 +227,7 @@ function renderLogLevelForm (template, responseData){
};
if (!responseData) {
var topologyId = $.url("?id");
- $.get ('/api/v1/topology/' + topologyId + '/logconfig', renderImpl);
+ getStatic ('/api/v1/topology/' + topologyId + '/logconfig', renderImpl);
} else {
renderImpl (responseData);
}
@@ -261,7 +261,7 @@ $(document).ready(function() {
$.ajaxSetup({
"error":function(jqXHR,textStatus,response) {
var errorJson = jQuery.parseJSON(jqXHR.responseText);
- $.get("/templates/json-error-template.html", function(template) {
+ getStatic("/templates/json-error-template.html", function(template) {
$("#json-response-error").append(Mustache.render($(template).filter("#json-error-template").html(),errorJson));
});
}
@@ -269,7 +269,7 @@ $(document).ready(function() {
$.getJSON(url,function(response,status,jqXHR) {
var uiUser = $("#ui-user");
- $.get("/templates/user-template.html", function(template) {
+ getStatic("/templates/user-template.html", function(template) {
uiUser.append(Mustache.render($(template).filter("#user-template").html(),response));
$('#ui-user [data-toggle="tooltip"]').tooltip();
});
http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-dist/binary/pom.xml
----------------------------------------------------------------------
diff --git a/storm-dist/binary/pom.xml b/storm-dist/binary/pom.xml
index a26f855..f7c4cdb 100644
--- a/storm-dist/binary/pom.xml
+++ b/storm-dist/binary/pom.xml
@@ -30,6 +30,16 @@
<name>Storm Binary Distribution</name>
<description>Storm binary distribution</description>
+ <!--
+ Used for cache busting in the Storm UI HTML and script.js files
+ See src/main/assembly/pom.xml for the fileSet rules with
+ <filtered>true</filtered>
+ -->
+ <properties>
+ <packageTimestamp>${maven.build.timestamp}</packageTimestamp>
+ <maven.build.timestamp.format>YYYYMMddHHmm</maven.build.timestamp.format>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
http://git-wip-us.apache.org/repos/asf/storm/blob/28ef601f/storm-dist/binary/src/main/assembly/binary.xml
----------------------------------------------------------------------
diff --git a/storm-dist/binary/src/main/assembly/binary.xml b/storm-dist/binary/src/main/assembly/binary.xml
index d524530..2333168 100644
--- a/storm-dist/binary/src/main/assembly/binary.xml
+++ b/storm-dist/binary/src/main/assembly/binary.xml
@@ -46,13 +46,36 @@
</includes>
<fileMode>0755</fileMode>
</fileSet>
+ <!--
+ Allow for variable substitution for cache busting in HTML files and script.js
+ See storm-dist/binary/pom.xml for custom variables that will be substituted in.
+
+ The source files should have a ${variable to be replaced} wherever
+ maven assembly plugin should inject a value.
+ -->
<fileSet>
<directory>${project.basedir}/../../storm-core/src/ui/public</directory>
<outputDirectory>public</outputDirectory>
<includes>
- <include>*/**</include>
+ <include>*.html</include>
+ <include>js/script.js</include>
</includes>
<excludes/>
+ <filtered>true</filtered>
+ </fileSet>
+ <!--
+ Include rest of public/ directory, without any filtering
+ -->
+ <fileSet>
+ <directory>${project.basedir}/../../storm-core/src/ui/public</directory>
+ <outputDirectory>public</outputDirectory>
+ <includes>
+ <include>*/**</include>
+ </includes>
+ <excludes>
+ <exclude>*.html</exclude>
+ <exclude>js/script.js</exclude>
+ </excludes>
</fileSet>
<fileSet>
<directory>${project.basedir}/../../examples</directory>
[08/13] storm git commit: Merge branch 'STORM-1964' of
github.com:arunmahadevan/storm
Posted by pt...@apache.org.
Merge branch 'STORM-1964' of github.com:arunmahadevan/storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a97b1628
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a97b1628
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a97b1628
Branch: refs/heads/master
Commit: a97b16285afe6a12f63360769985812fe7b5526f
Parents: e5be986 3cbd26e
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jul 21 13:21:24 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 13:21:24 2016 -0400
----------------------------------------------------------------------
.../storm/topology/WindowedBoltExecutor.java | 2 +-
.../storm/windowing/CountEvictionPolicy.java | 12 ++--
.../storm/windowing/CountTriggerPolicy.java | 2 +-
.../storm/windowing/DefaultEvictionContext.java | 53 +++++++++++++++
.../apache/storm/windowing/EvictionContext.java | 47 +++++++++++++
.../apache/storm/windowing/EvictionPolicy.java | 3 +-
.../storm/windowing/TimeEvictionPolicy.java | 4 +-
.../storm/windowing/TimeTriggerPolicy.java | 2 +-
.../windowing/WatermarkCountEvictionPolicy.java | 27 +++++---
.../windowing/WatermarkCountTriggerPolicy.java | 2 +-
.../windowing/WatermarkTimeTriggerPolicy.java | 3 +-
.../storm/windowing/WindowManagerTest.java | 69 ++++++++++++++++++--
12 files changed, 197 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
[07/13] storm git commit: Merge branch '1.0.x-branch' of
https://git-wip-us.apache.org/repos/asf/storm into 1.0.x-branch
Posted by pt...@apache.org.
Merge branch '1.0.x-branch' of https://git-wip-us.apache.org/repos/asf/storm into 1.0.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b56a34ab
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b56a34ab
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b56a34ab
Branch: refs/heads/1.0.x-branch
Commit: b56a34ab28f6849503d386f5d2e7b97ffb7f9603
Parents: b068dde 7936de9
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jul 21 12:25:33 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jul 21 12:25:33 2016 -0400
----------------------------------------------------------------------
CHANGELOG.md | 3 ++
.../apache/storm/command/shell_submission.clj | 3 +-
.../src/clj/org/apache/storm/daemon/nimbus.clj | 21 +++--------
.../src/clj/org/apache/storm/zookeeper.clj | 37 +++++++++++++++-----
.../org/apache/storm/utils/DisruptorQueue.java | 16 +++++++--
.../test/clj/org/apache/storm/nimbus_test.clj | 29 ---------------
6 files changed, 51 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b56a34ab/CHANGELOG.md
----------------------------------------------------------------------
diff --cc CHANGELOG.md
index 4182f1a,af5fcff..0b7173a
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@@ -1,5 -1,7 +1,8 @@@
## 1.0.2
+ * STORM-1977: Restore logic: give up leadership when elected as leader but doesn't have one or more topology codes on local
+ * STORM-1890 ensure we refetch static resources after package build
+ * STORM-1730: LocalCluster#shutdown() does not terminate all storm threds/thread pools.
+ * STORM-1976: STORM-1976 Remove cleanup-corrupt-topologies!
* STORM-1939: Frequent InterruptedException raised by ShellBoltMessageQueue.poll
* STORM-1968: Storm logviewer does not work for nimbus.log in secure cluster
* STORM-1928: ShellSpout should check heartbeat while ShellSpout is waiting for subprocess to sync