You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tg...@apache.org on 2017/03/16 22:45:25 UTC

[2/2] beam git commit: Populate Display Data in Window.Assign

Populate Display Data in Window.Assign

This populates the display data by delegating back to the enclosing
Window.Bound transform.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b5641d6f
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b5641d6f
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b5641d6f

Branch: refs/heads/master
Commit: b5641d6f72ec34a4ed001bef2a6bac0ebc4abcdc
Parents: a4142a3
Author: Thomas Groh <tg...@google.com>
Authored: Wed Mar 15 16:52:48 2017 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Thu Mar 16 15:45:14 2017 -0700

----------------------------------------------------------------------
 .../beam/sdk/transforms/windowing/Window.java   | 14 +++--
 .../sdk/transforms/windowing/WindowTest.java    | 54 ++++++++++++++++++++
 2 files changed, 65 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/b5641d6f/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
index 94870ff..373bbbd 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.transforms.windowing;
 
+import com.google.common.annotations.VisibleForTesting;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.annotations.Experimental.Kind;
@@ -479,7 +480,7 @@ public class Window {
             .setWindowingStrategyInternal(outputStrategy);
       } else {
         // This is the AssignWindows primitive
-        return input.apply(new Assign<T>(outputStrategy));
+        return input.apply(new Assign<>(this, outputStrategy));
       }
     }
 
@@ -532,11 +533,11 @@ public class Window {
     }
   }
 
-
   /**
    * A Primitive {@link PTransform} that assigns windows to elements based on a {@link WindowFn}.
    */
   public static class Assign<T> extends PTransform<PCollection<T>, PCollection<T>> {
+    private final Bound<T> original;
     private final WindowingStrategy<T, ?> updatedStrategy;
 
     /**
@@ -544,7 +545,9 @@ public class Window {
      * WindowingStrategy}. Windows should be assigned using the {@link WindowFn} returned by
      * {@link #getWindowFn()}.
      */
-    private Assign(WindowingStrategy updatedStrategy) {
+    @VisibleForTesting
+    Assign(Bound<T> original, WindowingStrategy updatedStrategy) {
+      this.original = original;
       this.updatedStrategy = updatedStrategy;
     }
 
@@ -554,6 +557,11 @@ public class Window {
           input.getPipeline(), updatedStrategy, input.isBounded());
     }
 
+    @Override
+    public void populateDisplayData(DisplayData.Builder builder) {
+      original.populateDisplayData(builder);
+    }
+
     public WindowFn<T, ?> getWindowFn() {
       return updatedStrategy.getWindowFn();
     }

http://git-wip-us.apache.org/repos/asf/beam/blob/b5641d6f/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
index 1101ebc..70635e9 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
@@ -30,6 +30,7 @@ import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.when;
 
+import com.google.common.collect.Iterables;
 import java.io.Serializable;
 import java.util.Collection;
 import java.util.Collections;
@@ -50,6 +51,8 @@ import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.SimpleFunction;
 import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+import org.apache.beam.sdk.transforms.windowing.Window.Bound;
 import org.apache.beam.sdk.util.WindowingStrategy;
 import org.apache.beam.sdk.util.WindowingStrategy.AccumulationMode;
 import org.apache.beam.sdk.values.KV;
@@ -446,6 +449,57 @@ public class WindowTest implements Serializable {
   }
 
   @Test
+  @Category(RunnableOnService.class)
+  public void testPrimitiveDisplayData() {
+    FixedWindows windowFn = FixedWindows.of(Duration.standardHours(5));
+    AfterWatermark.FromEndOfWindow triggerBuilder = AfterWatermark.pastEndOfWindow();
+    Duration allowedLateness = Duration.standardMinutes(10);
+    Window.ClosingBehavior closingBehavior = Window.ClosingBehavior.FIRE_IF_NON_EMPTY;
+    OutputTimeFn<BoundedWindow> outputTimeFn = OutputTimeFns.outputAtEndOfWindow();
+
+    Window.Bound<?> window = Window
+        .into(windowFn)
+        .triggering(triggerBuilder)
+        .accumulatingFiredPanes()
+        .withAllowedLateness(allowedLateness, closingBehavior)
+        .withOutputTimeFn(outputTimeFn);
+
+    DisplayData primitiveDisplayData =
+        Iterables.getOnlyElement(
+            DisplayDataEvaluator.create().displayDataForPrimitiveTransforms(window));
+
+    assertThat(primitiveDisplayData, hasDisplayItem("windowFn", windowFn.getClass()));
+    assertThat(primitiveDisplayData, includesDisplayDataFor("windowFn", windowFn));
+
+    assertThat(primitiveDisplayData, hasDisplayItem("trigger", triggerBuilder.toString()));
+    assertThat(primitiveDisplayData,
+        hasDisplayItem("accumulationMode", AccumulationMode.ACCUMULATING_FIRED_PANES.toString()));
+    assertThat(primitiveDisplayData,
+        hasDisplayItem("allowedLateness", allowedLateness));
+    assertThat(primitiveDisplayData, hasDisplayItem("closingBehavior", closingBehavior.toString()));
+    assertThat(primitiveDisplayData, hasDisplayItem("outputTimeFn", outputTimeFn.getClass()));
+  }
+
+  @Test
+  public void testAssignDisplayDataUnchanged() {
+    FixedWindows windowFn = FixedWindows.of(Duration.standardHours(5));
+
+    Bound<Object> original = Window.into(windowFn);
+    WindowingStrategy<?, ?> updated = WindowingStrategy.globalDefault().withWindowFn(windowFn);
+
+    DisplayData displayData = DisplayData.from(new Window.Assign<>(original, updated));
+
+    assertThat(displayData, hasDisplayItem("windowFn", windowFn.getClass()));
+    assertThat(displayData, includesDisplayDataFor("windowFn", windowFn));
+
+    assertThat(displayData, not(hasDisplayItem("trigger")));
+    assertThat(displayData, not(hasDisplayItem("accumulationMode")));
+    assertThat(displayData, not(hasDisplayItem("allowedLateness")));
+    assertThat(displayData, not(hasDisplayItem("closingBehavior")));
+    assertThat(displayData, not(hasDisplayItem("outputTimeFn")));
+  }
+
+  @Test
   public void testDisplayDataExcludesUnspecifiedProperties() {
     Window.Bound<?> onlyHasAccumulationMode = Window.discardingFiredPanes();
     assertThat(DisplayData.from(onlyHasAccumulationMode), not(hasDisplayItem(hasKey(isOneOf(