You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by bc...@apache.org on 2016/05/11 16:31:08 UTC

[1/9] incubator-beam git commit: Fix BigQuery sink display data

Repository: incubator-beam
Updated Branches:
  refs/heads/master acb040684 -> 9f796e22f


Fix BigQuery sink display data


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/489298fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/489298fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/489298fc

Branch: refs/heads/master
Commit: 489298fcd0371f70d0d8d20ece609618808a5f0c
Parents: d9355c2
Author: Scott Wegner <sw...@google.com>
Authored: Fri May 6 16:46:40 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700

----------------------------------------------------------------------
 .../dataflow/io/DataflowBigQueryIOTest.java     | 71 ++++++++++++++++++++
 .../java/org/apache/beam/sdk/io/BigQueryIO.java | 26 +++++++
 2 files changed, 97 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/489298fc/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowBigQueryIOTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowBigQueryIOTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowBigQueryIOTest.java
new file mode 100644
index 0000000..619da04
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowBigQueryIOTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow.io;
+
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasKey;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
+import org.apache.beam.runners.dataflow.transforms.DataflowDisplayDataEvaluator;
+import org.apache.beam.sdk.io.BigQueryIO;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+
+import com.google.api.services.bigquery.model.TableSchema;
+
+import org.junit.Test;
+
+import java.util.Set;
+
+/**
+ * Unit tests for Dataflow usage of {@link BigQueryIO} transforms.
+ */
+public class DataflowBigQueryIOTest {
+  @Test
+  public void testBatchSinkPrimitiveDisplayData() {
+    DataflowPipelineOptions options = DataflowDisplayDataEvaluator.getDefaultOptions();
+    options.setStreaming(false);
+    testSinkPrimitiveDisplayData(options);
+  }
+
+  @Test
+  public void testStreamingSinkPrimitiveDisplayData() {
+    DataflowPipelineOptions options = DataflowDisplayDataEvaluator.getDefaultOptions();
+    options.setStreaming(true);
+    testSinkPrimitiveDisplayData(options);
+  }
+
+  private void testSinkPrimitiveDisplayData(DataflowPipelineOptions options) {
+    DisplayDataEvaluator evaluator = DataflowDisplayDataEvaluator.create(options);
+
+    BigQueryIO.Write.Bound write = BigQueryIO.Write
+        .to("project:dataset.table")
+        .withSchema(new TableSchema().set("col1", "type1").set("col2", "type2"))
+        .withoutValidation();
+
+    Set<DisplayData> displayData = evaluator.displayDataForPrimitiveTransforms(write);
+    assertThat("BigQueryIO.Write should include the table spec in its primitive display data",
+        displayData, hasItem(hasDisplayItem(hasKey("tableSpec"))));
+
+    assertThat("BigQueryIO.Write should include the table schema in its primitive display data",
+        displayData, hasItem(hasDisplayItem(hasKey("schema"))));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/489298fc/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BigQueryIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BigQueryIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BigQueryIO.java
index 7785298..15872e5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BigQueryIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BigQueryIO.java
@@ -1166,6 +1166,15 @@ public class BigQueryIO {
       return new BigQueryWriteOperation(this);
     }
 
+    @Override
+    public void populateDisplayData(DisplayData.Builder builder) {
+      super.populateDisplayData(builder);
+
+      builder
+          .addIfNotNull(DisplayData.item("schema", jsonSchema))
+          .addIfNotNull(DisplayData.item("tableSpec", jsonTable));
+    }
+
     private static class BigQueryWriteOperation extends FileBasedWriteOperation<TableRow> {
       // The maximum number of retry load jobs.
       private static final int MAX_RETRY_LOAD_JOBS = 3;
@@ -1396,6 +1405,13 @@ public class BigQueryIO {
       uniqueIdsForTableRows.clear();
     }
 
+    @Override
+    public void populateDisplayData(DisplayData.Builder builder) {
+      super.populateDisplayData(builder);
+
+      builder.addIfNotNull(DisplayData.item("schema", jsonTableSchema));
+    }
+
     public TableReference getOrCreateTable(BigQueryOptions options, String tableSpec)
         throws IOException {
       TableReference tableReference = parseTableSpec(tableSpec);
@@ -1603,6 +1619,16 @@ public class BigQueryIO {
           new TableRowInfo(context.element(), uniqueId)));
     }
 
+    @Override
+    public void populateDisplayData(DisplayData.Builder builder) {
+      super.populateDisplayData(builder);
+
+      builder.addIfNotNull(DisplayData.item("tableSpec", tableSpec));
+      if (tableRefFunction != null) {
+        builder.add(DisplayData.item("tableFn", tableRefFunction.getClass()));
+      }
+    }
+
     private String tableSpecFromWindow(BigQueryOptions options, BoundedWindow window) {
       if (tableSpec != null) {
         return tableSpec;


[5/9] incubator-beam git commit: Exclude JsonIgnore PipelineOptions from DisplayData

Posted by bc...@apache.org.
Exclude JsonIgnore PipelineOptions from DisplayData


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/43085f7e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/43085f7e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/43085f7e

Branch: refs/heads/master
Commit: 43085f7e53e693202ab91768a7f3688fc54f24ee
Parents: 0dab643
Author: Scott Wegner <sw...@google.com>
Authored: Tue May 3 14:10:59 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700

----------------------------------------------------------------------
 .../beam/sdk/options/PipelineOptionSpec.java    | 61 +++++---------------
 .../sdk/options/ProxyInvocationHandler.java     | 10 ++++
 .../options/PipelineOptionsReflectorTest.java   | 32 ++++++++++
 .../sdk/options/ProxyInvocationHandlerTest.java |  9 +++
 4 files changed, 65 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/43085f7e/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
index 71f9d46..9a88f70 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
@@ -17,73 +17,40 @@
  */
 package org.apache.beam.sdk.options;
 
-import com.google.common.base.MoreObjects;
-import com.google.common.base.Objects;
+import com.google.auto.value.AutoValue;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
 
 import java.lang.reflect.Method;
 
 /**
  * For internal use. Specification for an option defined in a {@link PipelineOptions} interface.
  */
-class PipelineOptionSpec {
-  private final Class<? extends PipelineOptions> clazz;
-  private final String name;
-  private final Method getter;
-
+@AutoValue
+abstract class PipelineOptionSpec {
   static PipelineOptionSpec of(Class<? extends PipelineOptions> clazz, String name, Method getter) {
-    return new PipelineOptionSpec(clazz, name, getter);
-  }
-
-  private PipelineOptionSpec(Class<? extends PipelineOptions> clazz, String name, Method getter) {
-    this.clazz = clazz;
-    this.name = name;
-    this.getter = getter;
+    return new AutoValue_PipelineOptionSpec(clazz, name, getter);
   }
 
   /**
    * The {@link PipelineOptions} interface which defines this {@link PipelineOptionSpec}.
    */
-  Class<? extends PipelineOptions> getDefiningInterface() {
-    return clazz;
-  }
+  abstract Class<? extends PipelineOptions> getDefiningInterface();
 
   /**
    * Name of the property.
    */
-  String getName() {
-    return name;
-  }
+  abstract String getName();
 
   /**
    * The getter method for this property.
    */
-  Method getGetterMethod() {
-    return getter;
-  }
+  abstract Method getGetterMethod();
 
-  @Override
-  public String toString() {
-    return MoreObjects.toStringHelper(this)
-        .add("definingInterface", getDefiningInterface())
-        .add("name", getName())
-        .add("getterMethod", getGetterMethod())
-        .toString();
-  }
-
-  @Override
-  public int hashCode() {
-    return Objects.hashCode(getDefiningInterface(), getName(), getGetterMethod());
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (!(obj instanceof PipelineOptionSpec)) {
-      return false;
-    }
-
-    PipelineOptionSpec that = (PipelineOptionSpec) obj;
-    return Objects.equal(this.getDefiningInterface(), that.getDefiningInterface())
-        && Objects.equal(this.getName(), that.getName())
-        && Objects.equal(this.getGetterMethod(), that.getGetterMethod());
+  /**
+   * Whether the option should be serialized. Uses the {@link JsonIgnore} annotation.
+   */
+  boolean shouldSerialize() {
+    return !getGetterMethod().isAnnotationPresent(JsonIgnore.class);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/43085f7e/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
index 745549f..159eb5b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
@@ -281,6 +281,12 @@ class ProxyInvocationHandler implements InvocationHandler {
       HashSet<PipelineOptionSpec> specs = new HashSet<>(optionsMap.get(option.getKey()));
 
       for (PipelineOptionSpec optionSpec : specs) {
+        if (!optionSpec.shouldSerialize()) {
+          // Options that are excluded for serialization (i.e. those with @JsonIgnore) are also
+          // excluded from display data. These options are generally not useful for display.
+          continue;
+        }
+
         Class<?> pipelineInterface = optionSpec.getDefiningInterface();
         if (type != null) {
           builder.add(DisplayData.item(option.getKey(), type, value)
@@ -304,6 +310,10 @@ class ProxyInvocationHandler implements InvocationHandler {
           .withNamespace(UnknownPipelineOptions.class));
       } else {
         for (PipelineOptionSpec spec : specs) {
+          if (!spec.shouldSerialize()) {
+            continue;
+          }
+
           Object value = getValueFromJson(jsonOption.getKey(), spec.getGetterMethod());
           DisplayData.Type type = DisplayData.inferType(value);
           if (type != null) {

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/43085f7e/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsReflectorTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsReflectorTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsReflectorTest.java
index 82f0329..8f801c7 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsReflectorTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsReflectorTest.java
@@ -20,12 +20,15 @@ package org.apache.beam.sdk.options;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.isOneOf;
 import static org.hamcrest.Matchers.not;
 
 import com.google.common.collect.ImmutableSet;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import org.hamcrest.FeatureMatcher;
 import org.hamcrest.Matcher;
 import org.hamcrest.Matchers;
@@ -130,6 +133,24 @@ public class PipelineOptionsReflectorTest {
   }
 
   @Test
+  public void testShouldSerialize() {
+    Set<PipelineOptionSpec> properties =
+        PipelineOptionsReflector.getOptionSpecs(JsonIgnoreOptions.class);
+
+    assertThat(properties, hasItem(allOf(hasName("notIgnored"), shouldSerialize())));
+    assertThat(properties, hasItem(allOf(hasName("ignored"), not(shouldSerialize()))));
+  }
+
+  interface JsonIgnoreOptions extends PipelineOptions {
+    String getNotIgnored();
+    void setNotIgnored(String value);
+
+    @JsonIgnore
+    String getIgnored();
+    void setIgnored(String value);
+  }
+
+  @Test
   public void testMultipleInputInterfaces() {
     Set<Class<? extends PipelineOptions>> interfaces =
         ImmutableSet.<Class<? extends PipelineOptions>>of(
@@ -193,4 +214,15 @@ public class PipelineOptionsReflectorTest {
       }
     };
   }
+
+  private static Matcher<PipelineOptionSpec> shouldSerialize() {
+    return new FeatureMatcher<PipelineOptionSpec, Boolean>(equalTo(true),
+        "should serialize", "shouldSerialize") {
+
+      @Override
+      protected Boolean featureValueOf(PipelineOptionSpec actual) {
+        return actual.shouldSerialize();
+      }
+    };
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/43085f7e/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java
index b53000d..228a6ba 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java
@@ -951,4 +951,13 @@ public class ProxyInvocationHandlerTest {
     String value = MAPPER.writeValueAsString(options);
     return MAPPER.readValue(value, PipelineOptions.class).as(kls);
   }
+
+  @Test
+  public void testDisplayDataExcludesJsonIgnoreOptions() {
+    IgnoredProperty options = PipelineOptionsFactory.as(IgnoredProperty.class);
+    options.setValue("foobar");
+
+    DisplayData data = DisplayData.from(options);
+    assertThat(data, not(hasDisplayItem(hasKey("value"))));
+  }
 }


[4/9] incubator-beam git commit: Test utility for display data in a pipeline runner

Posted by bc...@apache.org.
Test utility for display data in a pipeline runner

DisplayDataEvaluator is useful for validating how PTransform
display data is surfaced in the context of a Pipeline and runner.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c4b41800
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c4b41800
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c4b41800

Branch: refs/heads/master
Commit: c4b418002ab05ca98f06e33f6c3e30effb28f90d
Parents: acb0406
Author: Scott Wegner <sw...@google.com>
Authored: Tue May 3 09:28:21 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700

----------------------------------------------------------------------
 .../DataflowDisplayDataEvaluator.java           |  71 +++++++++
 .../display/DisplayDataEvaluator.java           | 146 +++++++++++++++++++
 .../display/DisplayDataEvaluatorTest.java       |  74 ++++++++++
 3 files changed, 291 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c4b41800/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowDisplayDataEvaluator.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowDisplayDataEvaluator.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowDisplayDataEvaluator.java
new file mode 100644
index 0000000..db42c77
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowDisplayDataEvaluator.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow.transforms;
+
+import org.apache.beam.runners.dataflow.DataflowPipelineRunner;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
+import org.apache.beam.sdk.options.GcpOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+import org.apache.beam.sdk.util.NoopCredentialFactory;
+import org.apache.beam.sdk.util.NoopPathValidator;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Factory methods for creating {@link DisplayDataEvaluator} instances against the
+ * {@link DataflowPipelineRunner}.
+ */
+public final class DataflowDisplayDataEvaluator {
+  /** Do not instantiate. */
+  private DataflowDisplayDataEvaluator() {}
+
+  /**
+   * Retrieve a set of default {@link DataflowPipelineOptions} which can be used to build
+   * dataflow pipelines for evaluating display data.
+   */
+  public static DataflowPipelineOptions getDefaultOptions() {
+    DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
+
+    options.setProject("foobar");
+    options.setTempLocation("gs://bucket/tmpLocation");
+    options.setFilesToStage(Lists.<String>newArrayList());
+
+    options.as(DataflowPipelineDebugOptions.class).setPathValidatorClass(NoopPathValidator.class);
+    options.as(GcpOptions.class).setCredentialFactoryClass(NoopCredentialFactory.class);
+
+    return options;
+  }
+
+  /**
+   * Create a {@link DisplayDataEvaluator} instance to evaluate pipeline display data against
+   * the {@link DataflowPipelineRunner}.
+   */
+  public static DisplayDataEvaluator create() {
+    return create(getDefaultOptions());
+  }
+
+  /**
+   * Create a {@link DisplayDataEvaluator} instance to evaluate pipeline display data against
+   * the {@link DataflowPipelineRunner} with the specified {@code options}.
+   */
+  public static DisplayDataEvaluator create(DataflowPipelineOptions options) {
+    return DisplayDataEvaluator.forRunner(DataflowPipelineRunner.class, options);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c4b41800/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java
new file mode 100644
index 0000000..6759003
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.display;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.PipelineRunner;
+import org.apache.beam.sdk.runners.TransformTreeNode;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.POutput;
+
+import com.google.common.collect.Sets;
+
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Test utilities to evaluate the {@link DisplayData} in the context of a {@link PipelineRunner}.
+ */
+public class DisplayDataEvaluator {
+  private final PipelineOptions options;
+
+  /**
+   * Create a new {@link DisplayDataEvaluator} using the specified {@link PipelineRunner} and
+   * default {@link PipelineOptions}.
+   */
+  public static DisplayDataEvaluator forRunner(Class<? extends PipelineRunner<?>> pipelineRunner) {
+    return forRunner(pipelineRunner, PipelineOptionsFactory.create());
+  }
+
+  /**
+   * Create a new {@link DisplayDataEvaluator} using the specified {@link PipelineRunner} and
+   * {@link PipelineOptions}.
+   */
+  public static DisplayDataEvaluator forRunner(
+      Class<? extends PipelineRunner<?>> pipelineRunner, PipelineOptions pipelineOptions) {
+    return new DisplayDataEvaluator(pipelineRunner, pipelineOptions);
+  }
+
+  private DisplayDataEvaluator(Class<? extends PipelineRunner<?>> runner, PipelineOptions options) {
+    this.options = options;
+    this.options.setRunner(runner);
+  }
+
+  /**
+   * Traverse the specified {@link PTransform}, collecting {@link DisplayData} registered on the
+   * inner primitive {@link PTransform PTransforms}.
+   *
+   * @return the set of {@link DisplayData} for primitive {@link PTransform PTransforms}.
+   */
+  public <InputT> Set<DisplayData> displayDataForPrimitiveTransforms(
+      final PTransform<? super PCollection<InputT>, ? extends POutput> root) {
+    return displayDataForPrimitiveTransforms(root, null);
+  }
+
+  /**
+   * Traverse the specified {@link PTransform}, collecting {@link DisplayData} registered on the
+   * inner primitive {@link PTransform PTransforms}.
+   *
+   * @param root The root {@link PTransform} to traverse
+   * @param inputCoder The coder to set for the {@link PTransform} input, or null to infer the
+   *                   default coder.
+   *
+   * @return the set of {@link DisplayData} for primitive {@link PTransform PTransforms}.
+   */
+  public <InputT> Set<DisplayData> displayDataForPrimitiveTransforms(
+    final PTransform<? super PCollection<InputT>, ? extends POutput> root,
+    Coder<InputT> inputCoder) {
+
+    Create.Values<InputT> input = Create.of();
+    if (inputCoder != null) {
+      input = input.withCoder(inputCoder);
+    }
+
+    Pipeline pipeline = Pipeline.create(options);
+    pipeline
+      .apply(input)
+      .apply(root);
+
+    PrimitiveDisplayDataPTransformVisitor visitor = new PrimitiveDisplayDataPTransformVisitor(root);
+    pipeline.traverseTopologically(visitor);
+    return visitor.getPrimitivesDisplayData();
+  }
+
+  /**
+   * Visits {@link PTransform PTransforms} in a pipeline, and collects {@link DisplayData} for
+   * each primitive transform under a given composite root.
+   */
+  private static class PrimitiveDisplayDataPTransformVisitor
+  extends Pipeline.PipelineVisitor.Defaults {
+    private final PTransform root;
+    private final Set<DisplayData> displayData;
+    private boolean shouldRecord = false;
+
+    PrimitiveDisplayDataPTransformVisitor(PTransform root) {
+      this.root = root;
+      this.displayData = Sets.newHashSet();
+    }
+
+    Set<DisplayData> getPrimitivesDisplayData() {
+      return displayData;
+    }
+
+    @Override
+    public CompositeBehavior enterCompositeTransform(TransformTreeNode node) {
+      if (Objects.equals(root, node.getTransform())) {
+        shouldRecord = true;
+      }
+
+      return CompositeBehavior.ENTER_TRANSFORM;
+    }
+
+    @Override
+    public void leaveCompositeTransform(TransformTreeNode node) {
+      if (Objects.equals(root, node.getTransform())) {
+        shouldRecord = false;
+      }
+    }
+
+    @Override
+    public void visitPrimitiveTransform(TransformTreeNode node) {
+      if (shouldRecord) {
+        displayData.add(DisplayData.from(node.getTransform()));
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c4b41800/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluatorTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluatorTest.java
new file mode 100644
index 0000000..f24c133
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluatorTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.display;
+
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.sdk.runners.DirectPipelineRunner;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.POutput;
+
+import org.junit.Test;
+
+import java.io.Serializable;
+import java.util.Set;
+
+/**
+ * Unit tests for {@link DisplayDataEvaluator}.
+ */
+public class DisplayDataEvaluatorTest implements Serializable {
+
+  @Test
+  public void testDisplayDataForPrimitiveTransforms() {
+    PTransform<? super PCollection<String>, ? super POutput> myTransform =
+        new PTransform<PCollection<String>, POutput> () {
+          @Override
+          public PCollection<String> apply(PCollection<String> input) {
+            return input.apply(ParDo.of(new DoFn<String, String>() {
+              @Override
+              public void processElement(ProcessContext c) throws Exception {
+                c.output(c.element());
+              }
+
+              @Override
+              public void populateDisplayData(DisplayData.Builder builder) {
+                builder.add(DisplayData.item("primitiveKey", "primitiveValue"));
+              }
+            }));
+          }
+
+          @Override
+          public void populateDisplayData(DisplayData.Builder builder) {
+            builder.add(DisplayData.item("compositeKey", "compositeValue"));
+          }
+        };
+
+    DisplayDataEvaluator evaluator = DisplayDataEvaluator.forRunner(DirectPipelineRunner.class);
+    Set<DisplayData> displayData = evaluator.displayDataForPrimitiveTransforms(myTransform);
+
+    assertThat(displayData, not(hasItem(hasDisplayItem("compositeKey", "compositeValue"))));
+    assertThat(displayData, hasItem(hasDisplayItem("primitiveKey", "primitiveValue")));
+  }
+}


[2/9] incubator-beam git commit: Exclude default Window.allowedLateness display data

Posted by bc...@apache.org.
Exclude default Window.allowedLateness display data


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d9355c28
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d9355c28
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d9355c28

Branch: refs/heads/master
Commit: d9355c286fd7f3e6f7069b34172c32554b57bac8
Parents: ea8019e
Author: Scott Wegner <sw...@google.com>
Authored: Fri May 6 16:03:12 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700

----------------------------------------------------------------------
 .../org/apache/beam/sdk/transforms/windowing/Window.java    | 9 +++++++--
 .../apache/beam/sdk/transforms/windowing/WindowTest.java    | 6 ++++--
 2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d9355c28/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
index e9b3437..b9dd451 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
@@ -601,8 +601,13 @@ public class Window {
       super.populateDisplayData(builder);
       builder
           .add(DisplayData.item("windowFn", windowFn.getClass()))
-          .include(windowFn)
-          .addIfNotNull(DisplayData.item("allowedLateness", allowedLateness));
+          .include(windowFn);
+
+      if (allowedLateness != null) {
+        builder.addIfNotDefault(DisplayData.item("allowedLateness", allowedLateness),
+            Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
+
+      }
 
       if (trigger != null && !(trigger instanceof DefaultTrigger)) {
         builder.add(DisplayData.item("trigger", trigger.toString()));

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d9355c28/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
index 8ad590d..885f549 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
@@ -285,11 +285,13 @@ public class WindowTest implements Serializable {
   }
 
   @Test
-  public void testDisplayDataExcludesDefaultTrigger() {
+  public void testDisplayDataExcludesDefaults() {
     Window.Bound<?> window = Window.into(new GlobalWindows())
-        .triggering(DefaultTrigger.of());
+        .triggering(DefaultTrigger.of())
+        .withAllowedLateness(Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
 
     DisplayData data = DisplayData.from(window);
     assertThat(data, not(hasDisplayItem(hasKey("trigger"))));
+    assertThat(data, not(hasDisplayItem(hasKey("allowedLateness"))));
   }
 }


[9/9] incubator-beam git commit: This closes #280

Posted by bc...@apache.org.
This closes #280


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9f796e22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9f796e22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9f796e22

Branch: refs/heads/master
Commit: 9f796e22ff99472a1ad6f83fdc8de2d7f16da969
Parents: acb0406 489298f
Author: bchambers <bc...@google.com>
Authored: Tue May 10 16:21:36 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:36 2016 -0700

----------------------------------------------------------------------
 .../dataflow/io/DataflowBigQueryIOTest.java     |  71 +++++++++
 .../runners/dataflow/io/DataflowTextIOTest.java |  23 ++-
 .../transforms/DataflowCombineTest.java         |  58 ++++++++
 .../DataflowDisplayDataEvaluator.java           |  71 +++++++++
 .../transforms/DataflowMapElementsTest.java     |  55 +++++++
 .../java/org/apache/beam/sdk/io/BigQueryIO.java |  26 ++++
 .../apache/beam/sdk/io/OffsetBasedSource.java   |   2 +-
 .../main/java/org/apache/beam/sdk/io/Write.java |   5 +
 .../beam/sdk/options/PipelineOptionSpec.java    |  61 ++------
 .../sdk/options/ProxyInvocationHandler.java     |  10 ++
 .../org/apache/beam/sdk/transforms/Combine.java |   2 +-
 .../apache/beam/sdk/transforms/MapElements.java |  12 ++
 .../beam/sdk/transforms/windowing/Window.java   |   9 +-
 .../options/PipelineOptionsReflectorTest.java   |  32 ++++
 .../sdk/options/ProxyInvocationHandlerTest.java |   9 ++
 .../beam/sdk/transforms/MapElementsTest.java    |  36 ++++-
 .../display/DisplayDataEvaluator.java           | 146 +++++++++++++++++++
 .../display/DisplayDataEvaluatorTest.java       |  74 ++++++++++
 .../sdk/transforms/windowing/WindowTest.java    |   6 +-
 19 files changed, 652 insertions(+), 56 deletions(-)
----------------------------------------------------------------------



[6/9] incubator-beam git commit: Fix Write transform primitive display data

Posted by bc...@apache.org.
Fix Write transform primitive display data


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ea8019ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ea8019ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ea8019ec

Branch: refs/heads/master
Commit: ea8019eced55023e060fd6472b8d69d15bf73839
Parents: 007526f
Author: Scott Wegner <sw...@google.com>
Authored: Fri May 6 11:40:47 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700

----------------------------------------------------------------------
 .../runners/dataflow/io/DataflowTextIOTest.java | 23 +++++++++++++++++++-
 .../main/java/org/apache/beam/sdk/io/Write.java |  5 +++++
 2 files changed, 27 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ea8019ec/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java
index bfc99e8..8ff7d0e 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java
@@ -17,11 +17,21 @@
  */
 package org.apache.beam.runners.dataflow.io;
 
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasValue;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.startsWith;
+import static org.junit.Assert.assertThat;
+
 import org.apache.beam.runners.dataflow.DataflowPipelineRunner;
 import org.apache.beam.runners.dataflow.testing.TestDataflowPipelineOptions;
+import org.apache.beam.runners.dataflow.transforms.DataflowDisplayDataEvaluator;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
 import org.apache.beam.sdk.util.GcsUtil;
 import org.apache.beam.sdk.util.TestCredential;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
@@ -41,13 +51,13 @@ import java.nio.channels.SeekableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.StandardOpenOption;
 import java.util.List;
+import java.util.Set;
 
 /**
  * {@link DataflowPipelineRunner} specific tests for TextIO Read and Write transforms.
  */
 @RunWith(JUnit4.class)
 public class DataflowTextIOTest {
-
   private TestDataflowPipelineOptions buildTestPipelineOptions() {
     TestDataflowPipelineOptions options =
         PipelineOptionsFactory.as(TestDataflowPipelineOptions.class);
@@ -116,4 +126,15 @@ public class DataflowTextIOTest {
   private void applyRead(Pipeline pipeline, String path) {
     pipeline.apply("Read(" + path + ")", TextIO.Read.from(path));
   }
+
+  @Test
+  public void testPrimitiveDisplayData() {
+    DisplayDataEvaluator evaluator = DataflowDisplayDataEvaluator.create();
+
+    TextIO.Write.Bound<?> write = TextIO.Write.to("foobar");
+
+    Set<DisplayData> displayData = evaluator.displayDataForPrimitiveTransforms(write);
+    assertThat("TextIO.Write should include the file prefix in its primitive display data",
+        displayData, hasItem(hasDisplayItem(hasValue(startsWith("foobar")))));
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ea8019ec/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
index 9cb026a..0f2dbf8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
@@ -204,6 +204,11 @@ public class Write {
                 c.output(result);
               }
             }
+
+            @Override
+            public void populateDisplayData(DisplayData.Builder builder) {
+              Write.Bound.this.populateDisplayData(builder);
+            }
           }).withSideInputs(writeOperationView))
           .setCoder(writeOperation.getWriterResultCoder());
 


[3/9] incubator-beam git commit: Add display data for MapElements transform

Posted by bc...@apache.org.
Add display data for MapElements transform


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/0dab643a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/0dab643a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/0dab643a

Branch: refs/heads/master
Commit: 0dab643ae3c7d275016f8b11e406af5d15e7f388
Parents: 9f6c27c
Author: Scott Wegner <sw...@google.com>
Authored: Tue May 3 11:16:51 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700

----------------------------------------------------------------------
 .../transforms/DataflowMapElementsTest.java     | 55 ++++++++++++++++++++
 .../apache/beam/sdk/transforms/MapElements.java | 12 +++++
 .../beam/sdk/transforms/MapElementsTest.java    | 36 ++++++++++++-
 3 files changed, 101 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dab643a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowMapElementsTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowMapElementsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowMapElementsTest.java
new file mode 100644
index 0000000..8a5e67d
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowMapElementsTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow.transforms;
+
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+
+import org.junit.Test;
+
+import java.io.Serializable;
+import java.util.Set;
+
+/**
+ * Unit tests for Dataflow usage of {@link MapElements} transforms.
+ */
+public class DataflowMapElementsTest implements Serializable {
+  @Test
+  public void testPrimitiveDisplayData() {
+    SimpleFunction<?, ?> mapFn = new SimpleFunction<Integer, Integer>() {
+      @Override
+      public Integer apply(Integer input) {
+        return input;
+      }
+    };
+
+    MapElements<?, ?> map = MapElements.via(mapFn);
+    DisplayDataEvaluator evaluator = DataflowDisplayDataEvaluator.create();
+
+    Set<DisplayData> displayData = evaluator.displayDataForPrimitiveTransforms(map);
+    assertThat("MapElements should include the mapFn in its primitive display data",
+        displayData, hasItem(hasDisplayItem("mapFn", mapFn.getClass())));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dab643a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
index 47c2f5d..29d1dde 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.transforms;
 
+import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TypeDescriptor;
 
@@ -108,6 +109,17 @@ extends PTransform<PCollection<InputT>, PCollection<OutputT>> {
       public void processElement(ProcessContext c) {
         c.output(fn.apply(c.element()));
       }
+
+      @Override
+      public void populateDisplayData(DisplayData.Builder builder) {
+        MapElements.this.populateDisplayData(builder);
+      }
     })).setTypeDescriptorInternal(outputType);
   }
+
+  @Override
+  public void populateDisplayData(DisplayData.Builder builder) {
+    super.populateDisplayData(builder);
+    builder.add(DisplayData.item("mapFn", fn.getClass()));
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dab643a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java
index 5446512..1e2c826 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java
@@ -17,12 +17,15 @@
  */
 package org.apache.beam.sdk.transforms;
 
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+
 import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assert.assertThat;
 
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TypeDescriptor;
@@ -103,7 +106,7 @@ public class MapElementsTest implements Serializable {
     assertThat(pipeline.getCoderRegistry().getDefaultCoder(output.getTypeDescriptor()),
         equalTo(pipeline.getCoderRegistry().getDefaultCoder(new TypeDescriptor<String>() {})));
 
-    // Make sure the pipelien runs too
+    // Make sure the pipeline runs too
     pipeline.run();
   }
 
@@ -118,12 +121,41 @@ public class MapElementsTest implements Serializable {
     pipeline.run();
   }
 
+  @Test
+  public void testSerializableFunctionDisplayData() {
+    SerializableFunction<Integer, Integer> serializableFn =
+        new SerializableFunction<Integer, Integer>() {
+          @Override
+          public Integer apply(Integer input) {
+            return input;
+          }
+        };
+
+    MapElements<?, ?> serializableMap = MapElements.via(serializableFn)
+        .withOutputType(TypeDescriptor.of(Integer.class));
+    assertThat(DisplayData.from(serializableMap),
+        hasDisplayItem("mapFn", serializableFn.getClass()));
+  }
+
+  @Test
+  public void testSimpleFunctionDisplayData() {
+    SimpleFunction<?, ?> simpleFn = new SimpleFunction<Integer, Integer>() {
+      @Override
+      public Integer apply(Integer input) {
+        return input;
+      }
+    };
+
+    MapElements<?, ?> simpleMap = MapElements.via(simpleFn);
+    assertThat(DisplayData.from(simpleMap), hasDisplayItem("mapFn", simpleFn.getClass()));
+  }
+
   static class VoidValues<K, V>
       extends PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Void>>> {
 
     @Override
     public PCollection<KV<K, Void>> apply(PCollection<KV<K, V>> input) {
-      return input.apply(MapElements.<KV<K, V>, KV<K, Void>>via(
+      return input.apply(MapElements.via(
           new SimpleFunction<KV<K, V>, KV<K, Void>>() {
             @Override
             public KV<K, Void> apply(KV<K, V> input) {


[8/9] incubator-beam git commit: Exclude OffsetBasedSource minBundleSize default

Posted by bc...@apache.org.
Exclude OffsetBasedSource minBundleSize default


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/007526f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/007526f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/007526f5

Branch: refs/heads/master
Commit: 007526f5e8d332b4b38d2403272689f40ae291e7
Parents: 43085f7
Author: Scott Wegner <sw...@google.com>
Authored: Tue May 3 14:13:00 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/007526f5/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java
index 126535a..1c033f8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java
@@ -207,7 +207,7 @@ public abstract class OffsetBasedSource<T> extends BoundedSource<T> {
   public void populateDisplayData(DisplayData.Builder builder) {
     super.populateDisplayData(builder);
     builder
-        .add(DisplayData.item("minBundleSize", minBundleSize))
+        .addIfNotDefault(DisplayData.item("minBundleSize", minBundleSize), 1L)
         .addIfNotDefault(DisplayData.item("startOffset", startOffset), 0L)
         .addIfNotDefault(DisplayData.item("endOffset", endOffset), Long.MAX_VALUE);
   }


[7/9] incubator-beam git commit: Fix Combine transform primitive display data

Posted by bc...@apache.org.
Fix Combine transform primitive display data


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9f6c27c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9f6c27c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9f6c27c5

Branch: refs/heads/master
Commit: 9f6c27c587960f2eeec28f1c6d082a3ebe37e191
Parents: c4b4180
Author: Scott Wegner <sw...@google.com>
Authored: Tue May 3 09:33:55 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700

----------------------------------------------------------------------
 .../transforms/DataflowCombineTest.java         | 58 ++++++++++++++++++++
 .../org/apache/beam/sdk/transforms/Combine.java |  2 +-
 2 files changed, 59 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9f6c27c5/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowCombineTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowCombineTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowCombineTest.java
new file mode 100644
index 0000000..3af0cae
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowCombineTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow.transforms;
+
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.CombineTest;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.POutput;
+
+import org.junit.Test;
+
+import java.util.Set;
+
+/**
+ * Unit tests for Dataflow usage of {@link Combine} transforms.
+ */
+public class DataflowCombineTest {
+  @Test
+  public void testCombinePerKeyPrimitiveDisplayData() {
+    DisplayDataEvaluator evaluator = DataflowDisplayDataEvaluator.create();
+
+    CombineTest.UniqueInts combineFn = new CombineTest.UniqueInts();
+    PTransform<PCollection<KV<Integer, Integer>>, ? extends POutput> combine =
+        Combine.perKey(combineFn);
+
+    Set<DisplayData> displayData = evaluator.displayDataForPrimitiveTransforms(combine,
+        KvCoder.of(VarIntCoder.of(), VarIntCoder.of()));
+
+    assertThat("Combine.perKey should include the combineFn in its primitive transform",
+        displayData, hasItem(hasDisplayItem("combineFn", combineFn.getClass())));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9f6c27c5/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
index 8a39c98..ffbaafa 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
@@ -1781,7 +1781,7 @@ public class Combine {
     public PCollection<KV<K, OutputT>> apply(PCollection<KV<K, InputT>> input) {
       return input
           .apply(GroupByKey.<K, InputT>create(fewKeys))
-          .apply(Combine.<K, InputT, OutputT>groupedValues(fn).withSideInputs(sideInputs));
+          .apply(Combine.groupedValues(fn, fnDisplayData).withSideInputs(sideInputs));
     }
 
     @Override