You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ie...@apache.org on 2019/05/10 09:47:11 UTC
[beam] branch spark-runner_structured-streaming updated (5cf3e1b ->
1d9155d)
This is an automated email from the ASF dual-hosted git repository.
iemejia pushed a change to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 5cf3e1b fixup Enable UsesFailureMessage category of tests
new ad22b68 Pass transform based doFnSchemaInformation in ParDo translation
new be79a86 Enable UsesSchema tests on ValidatesRunner
new 1d9155d fixup hadoop-format is not mandataory to run ValidatesRunner tests
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
runners/spark/build.gradle | 6 +++---
.../structuredstreaming/translation/batch/ParDoTranslatorBatch.java | 5 ++++-
2 files changed, 7 insertions(+), 4 deletions(-)
[beam] 03/03: fixup hadoop-format is not mandataory to run
ValidatesRunner tests
Posted by ie...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
iemejia pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 1d9155d6b66a506ef79a057df94188c095911836
Author: Ismaël Mejía <ie...@gmail.com>
AuthorDate: Fri May 10 11:36:23 2019 +0200
fixup hadoop-format is not mandataory to run ValidatesRunner tests
---
runners/spark/build.gradle | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/runners/spark/build.gradle b/runners/spark/build.gradle
index 46e5c25..9619c5e 100644
--- a/runners/spark/build.gradle
+++ b/runners/spark/build.gradle
@@ -178,7 +178,9 @@ task validatesStructuredStreamingRunnerBatch(type: Test) {
systemProperty "spark.ui.showConsoleProgress", "false"
classpath = configurations.validatesRunner
- testClassesDirs = files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) + files(project(":beam-sdks-java-io-hadoop-format").sourceSets.test.output.classesDirs) + files(project.sourceSets.test.output.classesDirs)
+ testClassesDirs += files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs)
+ testClassesDirs += files(project.sourceSets.test.output.classesDirs)
+
// Only one SparkContext may be running in a JVM (SPARK-2243)
forkEvery 1
maxParallelForks 4
[beam] 01/03: Pass transform based doFnSchemaInformation in ParDo
translation
Posted by ie...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
iemejia pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit ad22b6828cf7b78945e10cd60362c27a0447e66a
Author: Ismaël Mejía <ie...@gmail.com>
AuthorDate: Fri May 10 11:44:53 2019 +0200
Pass transform based doFnSchemaInformation in ParDo translation
---
.../structuredstreaming/translation/batch/ParDoTranslatorBatch.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java
index b16d7e9..400b025 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java
@@ -77,6 +77,9 @@ class ParDoTranslatorBatch<InputT, OutputT>
signature.stateDeclarations().size() > 0 || signature.timerDeclarations().size() > 0;
checkState(!stateful, "States and timers are not supported for the moment.");
+ DoFnSchemaInformation doFnSchemaInformation =
+ ParDoTranslation.getSchemaInformation(context.getCurrentTransform());
+
// Init main variables
Dataset<WindowedValue<InputT>> inputDataSet = context.getDataset(context.getInput());
Map<TupleTag<?>, PValue> outputs = context.getOutputs();
@@ -110,7 +113,7 @@ class ParDoTranslatorBatch<InputT, OutputT>
inputCoder,
outputCoderMap,
broadcastStateData,
- DoFnSchemaInformation.create());
+ doFnSchemaInformation);
Dataset<Tuple2<TupleTag<?>, WindowedValue<?>>> allOutputs =
inputDataSet.mapPartitions(doFnWrapper, EncoderHelpers.tuple2Encoder());
[beam] 02/03: Enable UsesSchema tests on ValidatesRunner
Posted by ie...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
iemejia pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit be79a86181f91d3e9b201bc3eafd412833a8cc72
Author: Ismaël Mejía <ie...@gmail.com>
AuthorDate: Fri May 10 11:45:59 2019 +0200
Enable UsesSchema tests on ValidatesRunner
---
runners/spark/build.gradle | 2 --
1 file changed, 2 deletions(-)
diff --git a/runners/spark/build.gradle b/runners/spark/build.gradle
index 02dfac7..46e5c25 100644
--- a/runners/spark/build.gradle
+++ b/runners/spark/build.gradle
@@ -207,8 +207,6 @@ task validatesStructuredStreamingRunnerBatch(type: Test) {
// Portability
excludeCategories 'org.apache.beam.sdk.testing.UsesImpulse'
excludeCategories 'org.apache.beam.sdk.testing.UsesCrossLanguageTransforms'
- // Schema
- excludeCategories 'org.apache.beam.sdk.testing.UsesSchema'
}
}