You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tg...@apache.org on 2017/03/16 22:45:25 UTC
[2/2] beam git commit: Populate Display Data in Window.Assign
Populate Display Data in Window.Assign
This populates the display data by delegating back to the enclosing
Window.Bound transform.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b5641d6f
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b5641d6f
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b5641d6f
Branch: refs/heads/master
Commit: b5641d6f72ec34a4ed001bef2a6bac0ebc4abcdc
Parents: a4142a3
Author: Thomas Groh <tg...@google.com>
Authored: Wed Mar 15 16:52:48 2017 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Thu Mar 16 15:45:14 2017 -0700
----------------------------------------------------------------------
.../beam/sdk/transforms/windowing/Window.java | 14 +++--
.../sdk/transforms/windowing/WindowTest.java | 54 ++++++++++++++++++++
2 files changed, 65 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/b5641d6f/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
index 94870ff..373bbbd 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.transforms.windowing;
+import com.google.common.annotations.VisibleForTesting;
import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
@@ -479,7 +480,7 @@ public class Window {
.setWindowingStrategyInternal(outputStrategy);
} else {
// This is the AssignWindows primitive
- return input.apply(new Assign<T>(outputStrategy));
+ return input.apply(new Assign<>(this, outputStrategy));
}
}
@@ -532,11 +533,11 @@ public class Window {
}
}
-
/**
* A Primitive {@link PTransform} that assigns windows to elements based on a {@link WindowFn}.
*/
public static class Assign<T> extends PTransform<PCollection<T>, PCollection<T>> {
+ private final Bound<T> original;
private final WindowingStrategy<T, ?> updatedStrategy;
/**
@@ -544,7 +545,9 @@ public class Window {
* WindowingStrategy}. Windows should be assigned using the {@link WindowFn} returned by
* {@link #getWindowFn()}.
*/
- private Assign(WindowingStrategy updatedStrategy) {
+ @VisibleForTesting
+ Assign(Bound<T> original, WindowingStrategy updatedStrategy) {
+ this.original = original;
this.updatedStrategy = updatedStrategy;
}
@@ -554,6 +557,11 @@ public class Window {
input.getPipeline(), updatedStrategy, input.isBounded());
}
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ original.populateDisplayData(builder);
+ }
+
public WindowFn<T, ?> getWindowFn() {
return updatedStrategy.getWindowFn();
}
http://git-wip-us.apache.org/repos/asf/beam/blob/b5641d6f/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
index 1101ebc..70635e9 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
@@ -30,6 +30,7 @@ import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
+import com.google.common.collect.Iterables;
import java.io.Serializable;
import java.util.Collection;
import java.util.Collections;
@@ -50,6 +51,8 @@ import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.SimpleFunction;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+import org.apache.beam.sdk.transforms.windowing.Window.Bound;
import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
import org.apache.beam.sdk.values.KV;
@@ -446,6 +449,57 @@ public class WindowTest implements Serializable {
}
@Test
+ @Category(RunnableOnService.class)
+ public void testPrimitiveDisplayData() {
+ FixedWindows windowFn = FixedWindows.of(Duration.standardHours(5));
+ AfterWatermark.FromEndOfWindow triggerBuilder = AfterWatermark.pastEndOfWindow();
+ Duration allowedLateness = Duration.standardMinutes(10);
+ Window.ClosingBehavior closingBehavior = Window.ClosingBehavior.FIRE_IF_NON_EMPTY;
+ OutputTimeFn<BoundedWindow> outputTimeFn = OutputTimeFns.outputAtEndOfWindow();
+
+ Window.Bound<?> window = Window
+ .into(windowFn)
+ .triggering(triggerBuilder)
+ .accumulatingFiredPanes()
+ .withAllowedLateness(allowedLateness, closingBehavior)
+ .withOutputTimeFn(outputTimeFn);
+
+ DisplayData primitiveDisplayData =
+ Iterables.getOnlyElement(
+ DisplayDataEvaluator.create().displayDataForPrimitiveTransforms(window));
+
+ assertThat(primitiveDisplayData, hasDisplayItem("windowFn", windowFn.getClass()));
+ assertThat(primitiveDisplayData, includesDisplayDataFor("windowFn", windowFn));
+
+ assertThat(primitiveDisplayData, hasDisplayItem("trigger", triggerBuilder.toString()));
+ assertThat(primitiveDisplayData,
+ hasDisplayItem("accumulationMode", AccumulationMode.ACCUMULATING_FIRED_PANES.toString()));
+ assertThat(primitiveDisplayData,
+ hasDisplayItem("allowedLateness", allowedLateness));
+ assertThat(primitiveDisplayData, hasDisplayItem("closingBehavior", closingBehavior.toString()));
+ assertThat(primitiveDisplayData, hasDisplayItem("outputTimeFn", outputTimeFn.getClass()));
+ }
+
+ @Test
+ public void testAssignDisplayDataUnchanged() {
+ FixedWindows windowFn = FixedWindows.of(Duration.standardHours(5));
+
+ Bound<Object> original = Window.into(windowFn);
+ WindowingStrategy<?, ?> updated = WindowingStrategy.globalDefault().withWindowFn(windowFn);
+
+ DisplayData displayData = DisplayData.from(new Window.Assign<>(original, updated));
+
+ assertThat(displayData, hasDisplayItem("windowFn", windowFn.getClass()));
+ assertThat(displayData, includesDisplayDataFor("windowFn", windowFn));
+
+ assertThat(displayData, not(hasDisplayItem("trigger")));
+ assertThat(displayData, not(hasDisplayItem("accumulationMode")));
+ assertThat(displayData, not(hasDisplayItem("allowedLateness")));
+ assertThat(displayData, not(hasDisplayItem("closingBehavior")));
+ assertThat(displayData, not(hasDisplayItem("outputTimeFn")));
+ }
+
+ @Test
public void testDisplayDataExcludesUnspecifiedProperties() {
Window.Bound<?> onlyHasAccumulationMode = Window.discardingFiredPanes();
assertThat(DisplayData.from(onlyHasAccumulationMode), not(hasDisplayItem(hasKey(isOneOf(