You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by bc...@apache.org on 2016/05/11 16:31:08 UTC
[1/9] incubator-beam git commit: Fix BigQuery sink display data
Repository: incubator-beam
Updated Branches:
refs/heads/master acb040684 -> 9f796e22f
Fix BigQuery sink display data
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/489298fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/489298fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/489298fc
Branch: refs/heads/master
Commit: 489298fcd0371f70d0d8d20ece609618808a5f0c
Parents: d9355c2
Author: Scott Wegner <sw...@google.com>
Authored: Fri May 6 16:46:40 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700
----------------------------------------------------------------------
.../dataflow/io/DataflowBigQueryIOTest.java | 71 ++++++++++++++++++++
.../java/org/apache/beam/sdk/io/BigQueryIO.java | 26 +++++++
2 files changed, 97 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/489298fc/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowBigQueryIOTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowBigQueryIOTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowBigQueryIOTest.java
new file mode 100644
index 0000000..619da04
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowBigQueryIOTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow.io;
+
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasKey;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
+import org.apache.beam.runners.dataflow.transforms.DataflowDisplayDataEvaluator;
+import org.apache.beam.sdk.io.BigQueryIO;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+
+import com.google.api.services.bigquery.model.TableSchema;
+
+import org.junit.Test;
+
+import java.util.Set;
+
+/**
+ * Unit tests for Dataflow usage of {@link BigQueryIO} transforms.
+ */
+public class DataflowBigQueryIOTest {
+ @Test
+ public void testBatchSinkPrimitiveDisplayData() {
+ DataflowPipelineOptions options = DataflowDisplayDataEvaluator.getDefaultOptions();
+ options.setStreaming(false);
+ testSinkPrimitiveDisplayData(options);
+ }
+
+ @Test
+ public void testStreamingSinkPrimitiveDisplayData() {
+ DataflowPipelineOptions options = DataflowDisplayDataEvaluator.getDefaultOptions();
+ options.setStreaming(true);
+ testSinkPrimitiveDisplayData(options);
+ }
+
+ private void testSinkPrimitiveDisplayData(DataflowPipelineOptions options) {
+ DisplayDataEvaluator evaluator = DataflowDisplayDataEvaluator.create(options);
+
+ BigQueryIO.Write.Bound write = BigQueryIO.Write
+ .to("project:dataset.table")
+ .withSchema(new TableSchema().set("col1", "type1").set("col2", "type2"))
+ .withoutValidation();
+
+ Set<DisplayData> displayData = evaluator.displayDataForPrimitiveTransforms(write);
+ assertThat("BigQueryIO.Write should include the table spec in its primitive display data",
+ displayData, hasItem(hasDisplayItem(hasKey("tableSpec"))));
+
+ assertThat("BigQueryIO.Write should include the table schema in its primitive display data",
+ displayData, hasItem(hasDisplayItem(hasKey("schema"))));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/489298fc/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BigQueryIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BigQueryIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BigQueryIO.java
index 7785298..15872e5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BigQueryIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BigQueryIO.java
@@ -1166,6 +1166,15 @@ public class BigQueryIO {
return new BigQueryWriteOperation(this);
}
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+
+ builder
+ .addIfNotNull(DisplayData.item("schema", jsonSchema))
+ .addIfNotNull(DisplayData.item("tableSpec", jsonTable));
+ }
+
private static class BigQueryWriteOperation extends FileBasedWriteOperation<TableRow> {
// The maximum number of retry load jobs.
private static final int MAX_RETRY_LOAD_JOBS = 3;
@@ -1396,6 +1405,13 @@ public class BigQueryIO {
uniqueIdsForTableRows.clear();
}
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+
+ builder.addIfNotNull(DisplayData.item("schema", jsonTableSchema));
+ }
+
public TableReference getOrCreateTable(BigQueryOptions options, String tableSpec)
throws IOException {
TableReference tableReference = parseTableSpec(tableSpec);
@@ -1603,6 +1619,16 @@ public class BigQueryIO {
new TableRowInfo(context.element(), uniqueId)));
}
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+
+ builder.addIfNotNull(DisplayData.item("tableSpec", tableSpec));
+ if (tableRefFunction != null) {
+ builder.add(DisplayData.item("tableFn", tableRefFunction.getClass()));
+ }
+ }
+
private String tableSpecFromWindow(BigQueryOptions options, BoundedWindow window) {
if (tableSpec != null) {
return tableSpec;
[5/9] incubator-beam git commit: Exclude JsonIgnore PipelineOptions
from DisplayData
Posted by bc...@apache.org.
Exclude JsonIgnore PipelineOptions from DisplayData
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/43085f7e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/43085f7e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/43085f7e
Branch: refs/heads/master
Commit: 43085f7e53e693202ab91768a7f3688fc54f24ee
Parents: 0dab643
Author: Scott Wegner <sw...@google.com>
Authored: Tue May 3 14:10:59 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700
----------------------------------------------------------------------
.../beam/sdk/options/PipelineOptionSpec.java | 61 +++++---------------
.../sdk/options/ProxyInvocationHandler.java | 10 ++++
.../options/PipelineOptionsReflectorTest.java | 32 ++++++++++
.../sdk/options/ProxyInvocationHandlerTest.java | 9 +++
4 files changed, 65 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/43085f7e/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
index 71f9d46..9a88f70 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptionSpec.java
@@ -17,73 +17,40 @@
*/
package org.apache.beam.sdk.options;
-import com.google.common.base.MoreObjects;
-import com.google.common.base.Objects;
+import com.google.auto.value.AutoValue;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
import java.lang.reflect.Method;
/**
* For internal use. Specification for an option defined in a {@link PipelineOptions} interface.
*/
-class PipelineOptionSpec {
- private final Class<? extends PipelineOptions> clazz;
- private final String name;
- private final Method getter;
-
+@AutoValue
+abstract class PipelineOptionSpec {
static PipelineOptionSpec of(Class<? extends PipelineOptions> clazz, String name, Method getter) {
- return new PipelineOptionSpec(clazz, name, getter);
- }
-
- private PipelineOptionSpec(Class<? extends PipelineOptions> clazz, String name, Method getter) {
- this.clazz = clazz;
- this.name = name;
- this.getter = getter;
+ return new AutoValue_PipelineOptionSpec(clazz, name, getter);
}
/**
* The {@link PipelineOptions} interface which defines this {@link PipelineOptionSpec}.
*/
- Class<? extends PipelineOptions> getDefiningInterface() {
- return clazz;
- }
+ abstract Class<? extends PipelineOptions> getDefiningInterface();
/**
* Name of the property.
*/
- String getName() {
- return name;
- }
+ abstract String getName();
/**
* The getter method for this property.
*/
- Method getGetterMethod() {
- return getter;
- }
+ abstract Method getGetterMethod();
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("definingInterface", getDefiningInterface())
- .add("name", getName())
- .add("getterMethod", getGetterMethod())
- .toString();
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(getDefiningInterface(), getName(), getGetterMethod());
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof PipelineOptionSpec)) {
- return false;
- }
-
- PipelineOptionSpec that = (PipelineOptionSpec) obj;
- return Objects.equal(this.getDefiningInterface(), that.getDefiningInterface())
- && Objects.equal(this.getName(), that.getName())
- && Objects.equal(this.getGetterMethod(), that.getGetterMethod());
+ /**
+ * Whether the option should be serialized. Uses the {@link JsonIgnore} annotation.
+ */
+ boolean shouldSerialize() {
+ return !getGetterMethod().isAnnotationPresent(JsonIgnore.class);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/43085f7e/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
index 745549f..159eb5b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ProxyInvocationHandler.java
@@ -281,6 +281,12 @@ class ProxyInvocationHandler implements InvocationHandler {
HashSet<PipelineOptionSpec> specs = new HashSet<>(optionsMap.get(option.getKey()));
for (PipelineOptionSpec optionSpec : specs) {
+ if (!optionSpec.shouldSerialize()) {
+ // Options that are excluded for serialization (i.e. those with @JsonIgnore) are also
+ // excluded from display data. These options are generally not useful for display.
+ continue;
+ }
+
Class<?> pipelineInterface = optionSpec.getDefiningInterface();
if (type != null) {
builder.add(DisplayData.item(option.getKey(), type, value)
@@ -304,6 +310,10 @@ class ProxyInvocationHandler implements InvocationHandler {
.withNamespace(UnknownPipelineOptions.class));
} else {
for (PipelineOptionSpec spec : specs) {
+ if (!spec.shouldSerialize()) {
+ continue;
+ }
+
Object value = getValueFromJson(jsonOption.getKey(), spec.getGetterMethod());
DisplayData.Type type = DisplayData.inferType(value);
if (type != null) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/43085f7e/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsReflectorTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsReflectorTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsReflectorTest.java
index 82f0329..8f801c7 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsReflectorTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsReflectorTest.java
@@ -20,12 +20,15 @@ package org.apache.beam.sdk.options;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.isOneOf;
import static org.hamcrest.Matchers.not;
import com.google.common.collect.ImmutableSet;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
import org.hamcrest.FeatureMatcher;
import org.hamcrest.Matcher;
import org.hamcrest.Matchers;
@@ -130,6 +133,24 @@ public class PipelineOptionsReflectorTest {
}
@Test
+ public void testShouldSerialize() {
+ Set<PipelineOptionSpec> properties =
+ PipelineOptionsReflector.getOptionSpecs(JsonIgnoreOptions.class);
+
+ assertThat(properties, hasItem(allOf(hasName("notIgnored"), shouldSerialize())));
+ assertThat(properties, hasItem(allOf(hasName("ignored"), not(shouldSerialize()))));
+ }
+
+ interface JsonIgnoreOptions extends PipelineOptions {
+ String getNotIgnored();
+ void setNotIgnored(String value);
+
+ @JsonIgnore
+ String getIgnored();
+ void setIgnored(String value);
+ }
+
+ @Test
public void testMultipleInputInterfaces() {
Set<Class<? extends PipelineOptions>> interfaces =
ImmutableSet.<Class<? extends PipelineOptions>>of(
@@ -193,4 +214,15 @@ public class PipelineOptionsReflectorTest {
}
};
}
+
+ private static Matcher<PipelineOptionSpec> shouldSerialize() {
+ return new FeatureMatcher<PipelineOptionSpec, Boolean>(equalTo(true),
+ "should serialize", "shouldSerialize") {
+
+ @Override
+ protected Boolean featureValueOf(PipelineOptionSpec actual) {
+ return actual.shouldSerialize();
+ }
+ };
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/43085f7e/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java
index b53000d..228a6ba 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/ProxyInvocationHandlerTest.java
@@ -951,4 +951,13 @@ public class ProxyInvocationHandlerTest {
String value = MAPPER.writeValueAsString(options);
return MAPPER.readValue(value, PipelineOptions.class).as(kls);
}
+
+ @Test
+ public void testDisplayDataExcludesJsonIgnoreOptions() {
+ IgnoredProperty options = PipelineOptionsFactory.as(IgnoredProperty.class);
+ options.setValue("foobar");
+
+ DisplayData data = DisplayData.from(options);
+ assertThat(data, not(hasDisplayItem(hasKey("value"))));
+ }
}
[4/9] incubator-beam git commit: Test utility for display data in a
pipeline runner
Posted by bc...@apache.org.
Test utility for display data in a pipeline runner
DisplayDataEvaluator is useful for validating how PTransform
display data is surfaced in the context of a Pipeline and runner.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c4b41800
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c4b41800
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c4b41800
Branch: refs/heads/master
Commit: c4b418002ab05ca98f06e33f6c3e30effb28f90d
Parents: acb0406
Author: Scott Wegner <sw...@google.com>
Authored: Tue May 3 09:28:21 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700
----------------------------------------------------------------------
.../DataflowDisplayDataEvaluator.java | 71 +++++++++
.../display/DisplayDataEvaluator.java | 146 +++++++++++++++++++
.../display/DisplayDataEvaluatorTest.java | 74 ++++++++++
3 files changed, 291 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c4b41800/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowDisplayDataEvaluator.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowDisplayDataEvaluator.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowDisplayDataEvaluator.java
new file mode 100644
index 0000000..db42c77
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowDisplayDataEvaluator.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow.transforms;
+
+import org.apache.beam.runners.dataflow.DataflowPipelineRunner;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions;
+import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
+import org.apache.beam.sdk.options.GcpOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+import org.apache.beam.sdk.util.NoopCredentialFactory;
+import org.apache.beam.sdk.util.NoopPathValidator;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Factory methods for creating {@link DisplayDataEvaluator} instances against the
+ * {@link DataflowPipelineRunner}.
+ */
+public final class DataflowDisplayDataEvaluator {
+ /** Do not instantiate. */
+ private DataflowDisplayDataEvaluator() {}
+
+ /**
+ * Retrieve a set of default {@link DataflowPipelineOptions} which can be used to build
+ * dataflow pipelines for evaluating display data.
+ */
+ public static DataflowPipelineOptions getDefaultOptions() {
+ DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
+
+ options.setProject("foobar");
+ options.setTempLocation("gs://bucket/tmpLocation");
+ options.setFilesToStage(Lists.<String>newArrayList());
+
+ options.as(DataflowPipelineDebugOptions.class).setPathValidatorClass(NoopPathValidator.class);
+ options.as(GcpOptions.class).setCredentialFactoryClass(NoopCredentialFactory.class);
+
+ return options;
+ }
+
+ /**
+ * Create a {@link DisplayDataEvaluator} instance to evaluate pipeline display data against
+ * the {@link DataflowPipelineRunner}.
+ */
+ public static DisplayDataEvaluator create() {
+ return create(getDefaultOptions());
+ }
+
+ /**
+ * Create a {@link DisplayDataEvaluator} instance to evaluate pipeline display data against
+ * the {@link DataflowPipelineRunner} with the specified {@code options}.
+ */
+ public static DisplayDataEvaluator create(DataflowPipelineOptions options) {
+ return DisplayDataEvaluator.forRunner(DataflowPipelineRunner.class, options);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c4b41800/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java
new file mode 100644
index 0000000..6759003
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluator.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.display;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.PipelineRunner;
+import org.apache.beam.sdk.runners.TransformTreeNode;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.POutput;
+
+import com.google.common.collect.Sets;
+
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Test utilities to evaluate the {@link DisplayData} in the context of a {@link PipelineRunner}.
+ */
+public class DisplayDataEvaluator {
+ private final PipelineOptions options;
+
+ /**
+ * Create a new {@link DisplayDataEvaluator} using the specified {@link PipelineRunner} and
+ * default {@link PipelineOptions}.
+ */
+ public static DisplayDataEvaluator forRunner(Class<? extends PipelineRunner<?>> pipelineRunner) {
+ return forRunner(pipelineRunner, PipelineOptionsFactory.create());
+ }
+
+ /**
+ * Create a new {@link DisplayDataEvaluator} using the specified {@link PipelineRunner} and
+ * {@link PipelineOptions}.
+ */
+ public static DisplayDataEvaluator forRunner(
+ Class<? extends PipelineRunner<?>> pipelineRunner, PipelineOptions pipelineOptions) {
+ return new DisplayDataEvaluator(pipelineRunner, pipelineOptions);
+ }
+
+ private DisplayDataEvaluator(Class<? extends PipelineRunner<?>> runner, PipelineOptions options) {
+ this.options = options;
+ this.options.setRunner(runner);
+ }
+
+ /**
+ * Traverse the specified {@link PTransform}, collecting {@link DisplayData} registered on the
+ * inner primitive {@link PTransform PTransforms}.
+ *
+ * @return the set of {@link DisplayData} for primitive {@link PTransform PTransforms}.
+ */
+ public <InputT> Set<DisplayData> displayDataForPrimitiveTransforms(
+ final PTransform<? super PCollection<InputT>, ? extends POutput> root) {
+ return displayDataForPrimitiveTransforms(root, null);
+ }
+
+ /**
+ * Traverse the specified {@link PTransform}, collecting {@link DisplayData} registered on the
+ * inner primitive {@link PTransform PTransforms}.
+ *
+ * @param root The root {@link PTransform} to traverse
+ * @param inputCoder The coder to set for the {@link PTransform} input, or null to infer the
+ * default coder.
+ *
+ * @return the set of {@link DisplayData} for primitive {@link PTransform PTransforms}.
+ */
+ public <InputT> Set<DisplayData> displayDataForPrimitiveTransforms(
+ final PTransform<? super PCollection<InputT>, ? extends POutput> root,
+ Coder<InputT> inputCoder) {
+
+ Create.Values<InputT> input = Create.of();
+ if (inputCoder != null) {
+ input = input.withCoder(inputCoder);
+ }
+
+ Pipeline pipeline = Pipeline.create(options);
+ pipeline
+ .apply(input)
+ .apply(root);
+
+ PrimitiveDisplayDataPTransformVisitor visitor = new PrimitiveDisplayDataPTransformVisitor(root);
+ pipeline.traverseTopologically(visitor);
+ return visitor.getPrimitivesDisplayData();
+ }
+
+ /**
+ * Visits {@link PTransform PTransforms} in a pipeline, and collects {@link DisplayData} for
+ * each primitive transform under a given composite root.
+ */
+ private static class PrimitiveDisplayDataPTransformVisitor
+ extends Pipeline.PipelineVisitor.Defaults {
+ private final PTransform root;
+ private final Set<DisplayData> displayData;
+ private boolean shouldRecord = false;
+
+ PrimitiveDisplayDataPTransformVisitor(PTransform root) {
+ this.root = root;
+ this.displayData = Sets.newHashSet();
+ }
+
+ Set<DisplayData> getPrimitivesDisplayData() {
+ return displayData;
+ }
+
+ @Override
+ public CompositeBehavior enterCompositeTransform(TransformTreeNode node) {
+ if (Objects.equals(root, node.getTransform())) {
+ shouldRecord = true;
+ }
+
+ return CompositeBehavior.ENTER_TRANSFORM;
+ }
+
+ @Override
+ public void leaveCompositeTransform(TransformTreeNode node) {
+ if (Objects.equals(root, node.getTransform())) {
+ shouldRecord = false;
+ }
+ }
+
+ @Override
+ public void visitPrimitiveTransform(TransformTreeNode node) {
+ if (shouldRecord) {
+ displayData.add(DisplayData.from(node.getTransform()));
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c4b41800/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluatorTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluatorTest.java
new file mode 100644
index 0000000..f24c133
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/display/DisplayDataEvaluatorTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.display;
+
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.sdk.runners.DirectPipelineRunner;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.POutput;
+
+import org.junit.Test;
+
+import java.io.Serializable;
+import java.util.Set;
+
+/**
+ * Unit tests for {@link DisplayDataEvaluator}.
+ */
+public class DisplayDataEvaluatorTest implements Serializable {
+
+ @Test
+ public void testDisplayDataForPrimitiveTransforms() {
+ PTransform<? super PCollection<String>, ? super POutput> myTransform =
+ new PTransform<PCollection<String>, POutput> () {
+ @Override
+ public PCollection<String> apply(PCollection<String> input) {
+ return input.apply(ParDo.of(new DoFn<String, String>() {
+ @Override
+ public void processElement(ProcessContext c) throws Exception {
+ c.output(c.element());
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ builder.add(DisplayData.item("primitiveKey", "primitiveValue"));
+ }
+ }));
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ builder.add(DisplayData.item("compositeKey", "compositeValue"));
+ }
+ };
+
+ DisplayDataEvaluator evaluator = DisplayDataEvaluator.forRunner(DirectPipelineRunner.class);
+ Set<DisplayData> displayData = evaluator.displayDataForPrimitiveTransforms(myTransform);
+
+ assertThat(displayData, not(hasItem(hasDisplayItem("compositeKey", "compositeValue"))));
+ assertThat(displayData, hasItem(hasDisplayItem("primitiveKey", "primitiveValue")));
+ }
+}
[2/9] incubator-beam git commit: Exclude default
Window.allowedLateness display data
Posted by bc...@apache.org.
Exclude default Window.allowedLateness display data
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d9355c28
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d9355c28
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d9355c28
Branch: refs/heads/master
Commit: d9355c286fd7f3e6f7069b34172c32554b57bac8
Parents: ea8019e
Author: Scott Wegner <sw...@google.com>
Authored: Fri May 6 16:03:12 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700
----------------------------------------------------------------------
.../org/apache/beam/sdk/transforms/windowing/Window.java | 9 +++++++--
.../apache/beam/sdk/transforms/windowing/WindowTest.java | 6 ++++--
2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d9355c28/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
index e9b3437..b9dd451 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/Window.java
@@ -601,8 +601,13 @@ public class Window {
super.populateDisplayData(builder);
builder
.add(DisplayData.item("windowFn", windowFn.getClass()))
- .include(windowFn)
- .addIfNotNull(DisplayData.item("allowedLateness", allowedLateness));
+ .include(windowFn);
+
+ if (allowedLateness != null) {
+ builder.addIfNotDefault(DisplayData.item("allowedLateness", allowedLateness),
+ Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
+
+ }
if (trigger != null && !(trigger instanceof DefaultTrigger)) {
builder.add(DisplayData.item("trigger", trigger.toString()));
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d9355c28/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
index 8ad590d..885f549 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/windowing/WindowTest.java
@@ -285,11 +285,13 @@ public class WindowTest implements Serializable {
}
@Test
- public void testDisplayDataExcludesDefaultTrigger() {
+ public void testDisplayDataExcludesDefaults() {
Window.Bound<?> window = Window.into(new GlobalWindows())
- .triggering(DefaultTrigger.of());
+ .triggering(DefaultTrigger.of())
+ .withAllowedLateness(Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
DisplayData data = DisplayData.from(window);
assertThat(data, not(hasDisplayItem(hasKey("trigger"))));
+ assertThat(data, not(hasDisplayItem(hasKey("allowedLateness"))));
}
}
[9/9] incubator-beam git commit: This closes #280
Posted by bc...@apache.org.
This closes #280
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9f796e22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9f796e22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9f796e22
Branch: refs/heads/master
Commit: 9f796e22ff99472a1ad6f83fdc8de2d7f16da969
Parents: acb0406 489298f
Author: bchambers <bc...@google.com>
Authored: Tue May 10 16:21:36 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:36 2016 -0700
----------------------------------------------------------------------
.../dataflow/io/DataflowBigQueryIOTest.java | 71 +++++++++
.../runners/dataflow/io/DataflowTextIOTest.java | 23 ++-
.../transforms/DataflowCombineTest.java | 58 ++++++++
.../DataflowDisplayDataEvaluator.java | 71 +++++++++
.../transforms/DataflowMapElementsTest.java | 55 +++++++
.../java/org/apache/beam/sdk/io/BigQueryIO.java | 26 ++++
.../apache/beam/sdk/io/OffsetBasedSource.java | 2 +-
.../main/java/org/apache/beam/sdk/io/Write.java | 5 +
.../beam/sdk/options/PipelineOptionSpec.java | 61 ++------
.../sdk/options/ProxyInvocationHandler.java | 10 ++
.../org/apache/beam/sdk/transforms/Combine.java | 2 +-
.../apache/beam/sdk/transforms/MapElements.java | 12 ++
.../beam/sdk/transforms/windowing/Window.java | 9 +-
.../options/PipelineOptionsReflectorTest.java | 32 ++++
.../sdk/options/ProxyInvocationHandlerTest.java | 9 ++
.../beam/sdk/transforms/MapElementsTest.java | 36 ++++-
.../display/DisplayDataEvaluator.java | 146 +++++++++++++++++++
.../display/DisplayDataEvaluatorTest.java | 74 ++++++++++
.../sdk/transforms/windowing/WindowTest.java | 6 +-
19 files changed, 652 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
[6/9] incubator-beam git commit: Fix Write transform primitive
display data
Posted by bc...@apache.org.
Fix Write transform primitive display data
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ea8019ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ea8019ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ea8019ec
Branch: refs/heads/master
Commit: ea8019eced55023e060fd6472b8d69d15bf73839
Parents: 007526f
Author: Scott Wegner <sw...@google.com>
Authored: Fri May 6 11:40:47 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700
----------------------------------------------------------------------
.../runners/dataflow/io/DataflowTextIOTest.java | 23 +++++++++++++++++++-
.../main/java/org/apache/beam/sdk/io/Write.java | 5 +++++
2 files changed, 27 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ea8019ec/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java
index bfc99e8..8ff7d0e 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java
@@ -17,11 +17,21 @@
*/
package org.apache.beam.runners.dataflow.io;
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasValue;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.startsWith;
+import static org.junit.Assert.assertThat;
+
import org.apache.beam.runners.dataflow.DataflowPipelineRunner;
import org.apache.beam.runners.dataflow.testing.TestDataflowPipelineOptions;
+import org.apache.beam.runners.dataflow.transforms.DataflowDisplayDataEvaluator;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
import org.apache.beam.sdk.util.GcsUtil;
import org.apache.beam.sdk.util.TestCredential;
import org.apache.beam.sdk.util.gcsfs.GcsPath;
@@ -41,13 +51,13 @@ import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.util.List;
+import java.util.Set;
/**
* {@link DataflowPipelineRunner} specific tests for TextIO Read and Write transforms.
*/
@RunWith(JUnit4.class)
public class DataflowTextIOTest {
-
private TestDataflowPipelineOptions buildTestPipelineOptions() {
TestDataflowPipelineOptions options =
PipelineOptionsFactory.as(TestDataflowPipelineOptions.class);
@@ -116,4 +126,15 @@ public class DataflowTextIOTest {
private void applyRead(Pipeline pipeline, String path) {
pipeline.apply("Read(" + path + ")", TextIO.Read.from(path));
}
+
+ @Test
+ public void testPrimitiveDisplayData() {
+ DisplayDataEvaluator evaluator = DataflowDisplayDataEvaluator.create();
+
+ TextIO.Write.Bound<?> write = TextIO.Write.to("foobar");
+
+ Set<DisplayData> displayData = evaluator.displayDataForPrimitiveTransforms(write);
+ assertThat("TextIO.Write should include the file prefix in its primitive display data",
+ displayData, hasItem(hasDisplayItem(hasValue(startsWith("foobar")))));
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ea8019ec/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
index 9cb026a..0f2dbf8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/Write.java
@@ -204,6 +204,11 @@ public class Write {
c.output(result);
}
}
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ Write.Bound.this.populateDisplayData(builder);
+ }
}).withSideInputs(writeOperationView))
.setCoder(writeOperation.getWriterResultCoder());
[3/9] incubator-beam git commit: Add display data for MapElements
transform
Posted by bc...@apache.org.
Add display data for MapElements transform
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/0dab643a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/0dab643a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/0dab643a
Branch: refs/heads/master
Commit: 0dab643ae3c7d275016f8b11e406af5d15e7f388
Parents: 9f6c27c
Author: Scott Wegner <sw...@google.com>
Authored: Tue May 3 11:16:51 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700
----------------------------------------------------------------------
.../transforms/DataflowMapElementsTest.java | 55 ++++++++++++++++++++
.../apache/beam/sdk/transforms/MapElements.java | 12 +++++
.../beam/sdk/transforms/MapElementsTest.java | 36 ++++++++++++-
3 files changed, 101 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dab643a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowMapElementsTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowMapElementsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowMapElementsTest.java
new file mode 100644
index 0000000..8a5e67d
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowMapElementsTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow.transforms;
+
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+
+import org.junit.Test;
+
+import java.io.Serializable;
+import java.util.Set;
+
+/**
+ * Unit tests for Dataflow usage of {@link MapElements} transforms.
+ */
+public class DataflowMapElementsTest implements Serializable {
+ @Test
+ public void testPrimitiveDisplayData() {
+ SimpleFunction<?, ?> mapFn = new SimpleFunction<Integer, Integer>() {
+ @Override
+ public Integer apply(Integer input) {
+ return input;
+ }
+ };
+
+ MapElements<?, ?> map = MapElements.via(mapFn);
+ DisplayDataEvaluator evaluator = DataflowDisplayDataEvaluator.create();
+
+ Set<DisplayData> displayData = evaluator.displayDataForPrimitiveTransforms(map);
+ assertThat("MapElements should include the mapFn in its primitive display data",
+ displayData, hasItem(hasDisplayItem("mapFn", mapFn.getClass())));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dab643a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
index 47c2f5d..29d1dde 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/MapElements.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.transforms;
+import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.TypeDescriptor;
@@ -108,6 +109,17 @@ extends PTransform<PCollection<InputT>, PCollection<OutputT>> {
public void processElement(ProcessContext c) {
c.output(fn.apply(c.element()));
}
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ MapElements.this.populateDisplayData(builder);
+ }
})).setTypeDescriptorInternal(outputType);
}
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder.add(DisplayData.item("mapFn", fn.getClass()));
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0dab643a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java
index 5446512..1e2c826 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/MapElementsTest.java
@@ -17,12 +17,15 @@
*/
package org.apache.beam.sdk.transforms;
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertThat;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.TypeDescriptor;
@@ -103,7 +106,7 @@ public class MapElementsTest implements Serializable {
assertThat(pipeline.getCoderRegistry().getDefaultCoder(output.getTypeDescriptor()),
equalTo(pipeline.getCoderRegistry().getDefaultCoder(new TypeDescriptor<String>() {})));
- // Make sure the pipelien runs too
+ // Make sure the pipeline runs too
pipeline.run();
}
@@ -118,12 +121,41 @@ public class MapElementsTest implements Serializable {
pipeline.run();
}
+ @Test
+ public void testSerializableFunctionDisplayData() {
+ SerializableFunction<Integer, Integer> serializableFn =
+ new SerializableFunction<Integer, Integer>() {
+ @Override
+ public Integer apply(Integer input) {
+ return input;
+ }
+ };
+
+ MapElements<?, ?> serializableMap = MapElements.via(serializableFn)
+ .withOutputType(TypeDescriptor.of(Integer.class));
+ assertThat(DisplayData.from(serializableMap),
+ hasDisplayItem("mapFn", serializableFn.getClass()));
+ }
+
+ @Test
+ public void testSimpleFunctionDisplayData() {
+ SimpleFunction<?, ?> simpleFn = new SimpleFunction<Integer, Integer>() {
+ @Override
+ public Integer apply(Integer input) {
+ return input;
+ }
+ };
+
+ MapElements<?, ?> simpleMap = MapElements.via(simpleFn);
+ assertThat(DisplayData.from(simpleMap), hasDisplayItem("mapFn", simpleFn.getClass()));
+ }
+
static class VoidValues<K, V>
extends PTransform<PCollection<KV<K, V>>, PCollection<KV<K, Void>>> {
@Override
public PCollection<KV<K, Void>> apply(PCollection<KV<K, V>> input) {
- return input.apply(MapElements.<KV<K, V>, KV<K, Void>>via(
+ return input.apply(MapElements.via(
new SimpleFunction<KV<K, V>, KV<K, Void>>() {
@Override
public KV<K, Void> apply(KV<K, V> input) {
[8/9] incubator-beam git commit: Exclude OffsetBasedSource
minBundleSize default
Posted by bc...@apache.org.
Exclude OffsetBasedSource minBundleSize default
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/007526f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/007526f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/007526f5
Branch: refs/heads/master
Commit: 007526f5e8d332b4b38d2403272689f40ae291e7
Parents: 43085f7
Author: Scott Wegner <sw...@google.com>
Authored: Tue May 3 14:13:00 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/007526f5/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java
index 126535a..1c033f8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/OffsetBasedSource.java
@@ -207,7 +207,7 @@ public abstract class OffsetBasedSource<T> extends BoundedSource<T> {
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
builder
- .add(DisplayData.item("minBundleSize", minBundleSize))
+ .addIfNotDefault(DisplayData.item("minBundleSize", minBundleSize), 1L)
.addIfNotDefault(DisplayData.item("startOffset", startOffset), 0L)
.addIfNotDefault(DisplayData.item("endOffset", endOffset), Long.MAX_VALUE);
}
[7/9] incubator-beam git commit: Fix Combine transform primitive
display data
Posted by bc...@apache.org.
Fix Combine transform primitive display data
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9f6c27c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9f6c27c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9f6c27c5
Branch: refs/heads/master
Commit: 9f6c27c587960f2eeec28f1c6d082a3ebe37e191
Parents: c4b4180
Author: Scott Wegner <sw...@google.com>
Authored: Tue May 3 09:33:55 2016 -0700
Committer: bchambers <bc...@google.com>
Committed: Tue May 10 16:21:27 2016 -0700
----------------------------------------------------------------------
.../transforms/DataflowCombineTest.java | 58 ++++++++++++++++++++
.../org/apache/beam/sdk/transforms/Combine.java | 2 +-
2 files changed, 59 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9f6c27c5/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowCombineTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowCombineTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowCombineTest.java
new file mode 100644
index 0000000..3af0cae
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowCombineTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow.transforms;
+
+import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.junit.Assert.assertThat;
+
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.CombineTest;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.POutput;
+
+import org.junit.Test;
+
+import java.util.Set;
+
+/**
+ * Unit tests for Dataflow usage of {@link Combine} transforms.
+ */
+public class DataflowCombineTest {
+ @Test
+ public void testCombinePerKeyPrimitiveDisplayData() {
+ DisplayDataEvaluator evaluator = DataflowDisplayDataEvaluator.create();
+
+ CombineTest.UniqueInts combineFn = new CombineTest.UniqueInts();
+ PTransform<PCollection<KV<Integer, Integer>>, ? extends POutput> combine =
+ Combine.perKey(combineFn);
+
+ Set<DisplayData> displayData = evaluator.displayDataForPrimitiveTransforms(combine,
+ KvCoder.of(VarIntCoder.of(), VarIntCoder.of()));
+
+ assertThat("Combine.perKey should include the combineFn in its primitive transform",
+ displayData, hasItem(hasDisplayItem("combineFn", combineFn.getClass())));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9f6c27c5/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
index 8a39c98..ffbaafa 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
@@ -1781,7 +1781,7 @@ public class Combine {
public PCollection<KV<K, OutputT>> apply(PCollection<KV<K, InputT>> input) {
return input
.apply(GroupByKey.<K, InputT>create(fewKeys))
- .apply(Combine.<K, InputT, OutputT>groupedValues(fn).withSideInputs(sideInputs));
+ .apply(Combine.groupedValues(fn, fnDisplayData).withSideInputs(sideInputs));
}
@Override