You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ec...@apache.org on 2018/11/27 15:39:10 UTC
[beam] branch spark-runner_structured-streaming created (now
358170a)
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a change to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git.
at 358170a Add precise TODO for multiple TransformTranslator per transform URN
This branch includes the following new commits:
new 03d333d Add an empty spark-structured-streaming runner project targeting spark 2.4.0
new 0a89b75 Fix missing dep
new 8b5c33c Add SparkPipelineOptions
new ad32304 Start pipeline translation
new 999761a Add global pipeline translation structure
new 91f283a Add nodes translators structure
new b77c7bb Wire node translators with pipeline translator
new b6b426e Renames: better differenciate pipeline translator for transform translator
new 5cb1add Organise methods in PipelineTranslator
new 53b2e71 Initialise BatchTranslationContext
new 3ce792d Refactoring: -move batch/streaming common translation visitor and utility methods to PipelineTranslator -rename batch dedicated classes to Batch* to differentiate with their streaming counterparts -Introduce TranslationContext for common batch/streaming components
new 8184de0 Make transform translation clearer: renaming, comments
new 3251ab2 Improve javadocs
new fc03a4a Move SparkTransformOverrides to correct package
new 41742be Move common translation context components to superclass
new 92ed130 apply spotless for e-formatting
new 74054fe Make codestyle and firebug happy
new af6a350 Add TODOs
new c567d40 Post-pone batch qualifier in all classes names for readability
new 358170a Add precise TODO for multiple TransformTranslator per transform URN
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[beam] 12/20: Make transform translation clearer: renaming, comments
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 8184de01ba7f1c75579e7ff496f064b579dcb362
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Thu Nov 22 11:06:15 2018 +0100
Make transform translation clearer: renaming, comments
---
.../translation/PipelineTranslator.java | 18 +++++++++++-------
.../translation/TransformTranslator.java | 2 +-
.../batch/BatchCombinePerKeyTranslator.java | 2 +-
.../batch/BatchFlattenPCollectionTranslator.java | 2 +-
.../translation/batch/BatchGroupByKeyTranslator.java | 2 +-
.../translation/batch/BatchParDoTranslator.java | 2 +-
.../translation/batch/BatchReadSourceTranslator.java | 2 +-
.../translation/batch/BatchReshuffleTranslator.java | 2 +-
.../translation/batch/BatchWindowAssignTranslator.java | 2 +-
9 files changed, 19 insertions(+), 15 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
index 62e87f2..185879b 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
@@ -123,19 +123,23 @@ public abstract class PipelineTranslator extends Pipeline.PipelineVisitor.Defaul
*/
protected abstract TransformTranslator<?> getTransformTranslator(TransformHierarchy.Node node);
- private <T extends PTransform<?, ?>> void translateNode(
+ /**
+ * Apply the given TransformTranslator to the given node.
+ */
+ private <T extends PTransform<?, ?>> void applyTransformTranslator(
TransformHierarchy.Node node,
TransformTranslator<?> transformTranslator) {
+ // create the applied PTransform on the translationContext
+ translationContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
+ // avoid type capture
@SuppressWarnings("unchecked")
T typedTransform = (T) node.getTransform();
-
@SuppressWarnings("unchecked")
TransformTranslator<T> typedTransformTranslator = (TransformTranslator<T>) transformTranslator;
- // create the applied PTransform on the translationContext
- translationContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
- typedTransformTranslator.translateNode(typedTransform, translationContext);
+ // apply the transformTranslator
+ typedTransformTranslator.translateTransform(typedTransform, translationContext);
}
@@ -165,7 +169,7 @@ public abstract class PipelineTranslator extends Pipeline.PipelineVisitor.Defaul
TransformTranslator<?> transformTranslator = getTransformTranslator(node);
if (transformTranslator != null) {
- translateNode(node, transformTranslator);
+ applyTransformTranslator(node, transformTranslator);
LOG.info("{} translated- {}", genSpaces(depth), node.getFullName());
return CompositeBehavior.DO_NOT_ENTER_TRANSFORM;
} else {
@@ -191,6 +195,6 @@ public abstract class PipelineTranslator extends Pipeline.PipelineVisitor.Defaul
throw new UnsupportedOperationException(
"The transform " + transformUrn + " is currently not supported.");
}
- translateNode(node, transformTranslator);
+ applyTransformTranslator(node, transformTranslator);
}
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
index 51cdd99..ebb8bf8 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
@@ -6,6 +6,6 @@ public interface TransformTranslator<TransformT extends PTransform> {
/** A translator of a {@link PTransform}. */
- void translateNode(TransformT transform, TranslationContext context);
+ void translateTransform(TransformT transform, TranslationContext context);
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchCombinePerKeyTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchCombinePerKeyTranslator.java
index c9cae47..858df18 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchCombinePerKeyTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchCombinePerKeyTranslator.java
@@ -9,7 +9,7 @@ import org.apache.beam.sdk.values.PCollection;
class BatchCombinePerKeyTranslator<K, InputT, AccumT, OutputT> implements
TransformTranslator<PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>>> {
- @Override public void translateNode(
+ @Override public void translateTransform(
PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>> transform,
TranslationContext context) {
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchFlattenPCollectionTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchFlattenPCollectionTranslator.java
index 77f6fdb..90c487a 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchFlattenPCollectionTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchFlattenPCollectionTranslator.java
@@ -9,7 +9,7 @@ import org.apache.beam.sdk.values.PCollectionList;
class BatchFlattenPCollectionTranslator<T> implements
TransformTranslator<PTransform<PCollectionList<T>, PCollection<T>>> {
- @Override public void translateNode(PTransform<PCollectionList<T>, PCollection<T>> transform,
+ @Override public void translateTransform(PTransform<PCollectionList<T>, PCollection<T>> transform,
TranslationContext context) {
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchGroupByKeyTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchGroupByKeyTranslator.java
index 1bd42f5..52a3c39 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchGroupByKeyTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchGroupByKeyTranslator.java
@@ -9,7 +9,7 @@ import org.apache.beam.sdk.values.PCollection;
class BatchGroupByKeyTranslator<K, InputT> implements
TransformTranslator<PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, Iterable<InputT>>>>> {
- @Override public void translateNode(
+ @Override public void translateTransform(
PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, Iterable<InputT>>>> transform,
TranslationContext context) {
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchParDoTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchParDoTranslator.java
index cf8c896..6e7f342 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchParDoTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchParDoTranslator.java
@@ -9,7 +9,7 @@ import org.apache.beam.sdk.values.PCollectionTuple;
class BatchParDoTranslator<InputT, OutputT> implements
TransformTranslator<PTransform<PCollection<InputT>, PCollectionTuple>> {
- @Override public void translateNode(PTransform<PCollection<InputT>, PCollectionTuple> transform,
+ @Override public void translateTransform(PTransform<PCollection<InputT>, PCollectionTuple> transform,
TranslationContext context) {
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReadSourceTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReadSourceTranslator.java
index f5f0351..4236b1c 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReadSourceTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReadSourceTranslator.java
@@ -8,7 +8,7 @@ import org.apache.beam.sdk.values.PCollection;
class BatchReadSourceTranslator<T> implements TransformTranslator<PTransform<PBegin, PCollection<T>>> {
- @Override public void translateNode(PTransform<PBegin, PCollection<T>> transform,
+ @Override public void translateTransform(PTransform<PBegin, PCollection<T>> transform,
TranslationContext context) {
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReshuffleTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReshuffleTranslator.java
index 5fab1c8..5baa331 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReshuffleTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReshuffleTranslator.java
@@ -6,7 +6,7 @@ import org.apache.beam.sdk.transforms.Reshuffle;
class BatchReshuffleTranslator<K, InputT> implements TransformTranslator<Reshuffle<K, InputT>> {
- @Override public void translateNode(Reshuffle<K, InputT> transform, TranslationContext context) {
+ @Override public void translateTransform(Reshuffle<K, InputT> transform, TranslationContext context) {
}
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchWindowAssignTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchWindowAssignTranslator.java
index fbbced5..1a8f68b 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchWindowAssignTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchWindowAssignTranslator.java
@@ -8,7 +8,7 @@ import org.apache.beam.sdk.values.PCollection;
class BatchWindowAssignTranslator<T> implements
TransformTranslator<PTransform<PCollection<T>, PCollection<T>>> {
- @Override public void translateNode(PTransform<PCollection<T>, PCollection<T>> transform,
+ @Override public void translateTransform(PTransform<PCollection<T>, PCollection<T>> transform,
TranslationContext context) {
}
}
[beam] 20/20: Add precise TODO for multiple TransformTranslator per
transform URN
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 358170a0200723cb7246074537931b2dd937a4e3
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Mon Nov 26 11:31:36 2018 +0100
Add precise TODO for multiple TransformTranslator per transform URN
---
.../translation/batch/PipelineTranslatorBatch.java | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java
index e883131..318d74c 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java
@@ -42,6 +42,12 @@ public class PipelineTranslatorBatch extends PipelineTranslator {
@SuppressWarnings("rawtypes")
private static final Map<String, TransformTranslator> TRANSFORM_TRANSLATORS = new HashMap<>();
+ //TODO the ability to have more than one TransformTranslator per URN
+ // that could be dynamically chosen by a predicated that evaluates based on PCollection
+ // obtainable though node.getInputs.getValue()
+ // See https://github.com/seznam/euphoria/blob/master/euphoria-spark/src/main/java/cz/seznam/euphoria/spark/SparkFlowTranslator.java#L83
+ // And https://github.com/seznam/euphoria/blob/master/euphoria-spark/src/main/java/cz/seznam/euphoria/spark/SparkFlowTranslator.java#L106
+
static {
TRANSFORM_TRANSLATORS.put(
PTransformTranslation.COMBINE_PER_KEY_TRANSFORM_URN, new CombinePerKeyTranslatorBatch());
[beam] 13/20: Improve javadocs
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 3251ab226cd43a381a9f1b4c9d5a4e145c2aa3a4
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Thu Nov 22 11:50:17 2018 +0100
Improve javadocs
---
.../spark/structuredstreaming/translation/PipelineTranslator.java | 8 ++++----
.../structuredstreaming/translation/TransformTranslator.java | 2 +-
.../spark/structuredstreaming/translation/TranslationContext.java | 4 ++++
.../translation/batch/BatchPipelineTranslator.java | 4 +++-
.../translation/batch/BatchTranslationContext.java | 2 +-
.../translation/streaming/StreamingPipelineTranslator.java | 5 +++++
.../translation/streaming/StreamingTranslationContext.java | 3 +++
7 files changed, 21 insertions(+), 7 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
index 185879b..51d65ff 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
@@ -14,10 +14,10 @@ import org.apache.beam.sdk.values.PValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- /**
- * The role of this class is to detect the pipeline mode and to translate the Beam operators to their Spark counterparts. If we have
- * a streaming job, this is instantiated as a {@link StreamingPipelineTranslator}. In other
- * case, i.e. for a batch job, a {@link BatchPipelineTranslator} is created. Correspondingly,
+ * {@link Pipeline.PipelineVisitor} that translates the Beam operators to their Spark counterparts.
+ * It also does the pipeline preparation: mode detection, transforms replacement, classpath preparation.
+ * If we have a streaming job, it is instantiated as a {@link StreamingPipelineTranslator}.
+ * If we have a batch job, it is instantiated as a {@link BatchPipelineTranslator}.
*/
public abstract class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
index ebb8bf8..54b0a85 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
@@ -4,7 +4,7 @@ import org.apache.beam.sdk.transforms.PTransform;
public interface TransformTranslator<TransformT extends PTransform> {
- /** A translator of a {@link PTransform}. */
+ /** Base class for translators of {@link PTransform}. */
void translateTransform(TransformT transform, TranslationContext context);
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
index 341ed49..3dacde4 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
@@ -1,7 +1,11 @@
package org.apache.beam.runners.spark.structuredstreaming.translation;
import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+/**
+ * Base class that gives a context for {@link PTransform} translation.
+ */
public class TranslationContext {
private AppliedPTransform<?, ?, ?> currentTransform;
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
index ff92d89..38324c0 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
@@ -11,7 +11,9 @@ import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.transforms.PTransform;
-/** {@link Pipeline.PipelineVisitor} for executing a {@link Pipeline} as a Spark batch job. */
+/** {@link PipelineTranslator} for executing a {@link Pipeline} in Spark in batch mode.
+ * This contains only the components specific to batch: {@link BatchTranslationContext},
+ * registry of batch {@link TransformTranslator} and registry lookup code. */
public class BatchPipelineTranslator extends PipelineTranslator {
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
index 71ef315..f08e33c 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
@@ -11,7 +11,7 @@ import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SparkSession;
/**
- * Keeps track of the {@link Dataset} and the step the translation is in.
+ * Keeps track of context of the translation.
*/
public class BatchTranslationContext extends TranslationContext {
private final Map<PValue, Dataset<?>> datasets;
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
index 9303d59..9cbfbed 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
@@ -3,8 +3,13 @@ package org.apache.beam.runners.spark.structuredstreaming.translation.streaming;
import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
import org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator;
import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
+import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.TransformHierarchy;
+/** {@link PipelineTranslator} for executing a {@link Pipeline} in Spark in streaming mode.
+ * This contains only the components specific to streaming: {@link StreamingTranslationContext},
+ * registry of batch {@link TransformTranslator} and registry lookup code. */
+
public class StreamingPipelineTranslator extends PipelineTranslator {
public StreamingPipelineTranslator(SparkPipelineOptions options) {
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java
index 460dbf6..f2ee34b 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java
@@ -2,6 +2,9 @@ package org.apache.beam.runners.spark.structuredstreaming.translation.streaming;
import org.apache.beam.runners.spark.structuredstreaming.translation.TranslationContext;
+/**
+ * * Keeps track of context of the translation.
+ */
public class StreamingTranslationContext extends TranslationContext {
}
[beam] 03/20: Add SparkPipelineOptions
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 8b5c33c20a852f0df691d1d8cca2ca9e10eaa0e6
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Wed Nov 14 15:24:28 2018 +0100
Add SparkPipelineOptions
---
.../structuredstreaming/SparkPipelineOptions.java | 106 +++++++++++++++++++++
1 file changed, 106 insertions(+)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineOptions.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineOptions.java
new file mode 100644
index 0000000..d381b5f
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineOptions.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark.structuredstreaming;
+
+import java.util.List;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.options.ApplicationNameOptions;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.DefaultValueFactory;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.StreamingOptions;
+
+/**
+ * Spark runner {@link PipelineOptions} handles Spark execution-related configurations, such as the
+ * master address, and other user-related knobs.
+ */
+public interface SparkPipelineOptions
+ extends PipelineOptions, StreamingOptions, ApplicationNameOptions {
+
+ @Description("The url of the spark master to connect to, (e.g. spark://host:port, local[4]).")
+ @Default.String("local[4]")
+ String getSparkMaster();
+
+ void setSparkMaster(String master);
+
+ @Description("Batch default storage level")
+ @Default.String("MEMORY_ONLY")
+ String getStorageLevel();
+
+ void setStorageLevel(String storageLevel);
+
+ @Description(
+ "A checkpoint directory for streaming resilience, ignored in batch. "
+ + "For durability, a reliable filesystem such as HDFS/S3/GS is necessary.")
+ @Default.InstanceFactory(TmpCheckpointDirFactory.class)
+ String getCheckpointDir();
+
+ void setCheckpointDir(String checkpointDir);
+
+ /**
+ * Returns the default checkpoint directory of /tmp/${job.name}. For testing purposes only.
+ * Production applications should use a reliable filesystem such as HDFS/S3/GS.
+ */
+ class TmpCheckpointDirFactory implements DefaultValueFactory<String> {
+ @Override
+ public String create(PipelineOptions options) {
+ return "/tmp/" + options.as(SparkPipelineOptions.class).getJobName();
+ }
+ }
+
+ @Description(
+ "The period to checkpoint (in Millis). If not set, Spark will default "
+ + "to Max(slideDuration, Seconds(10)). This PipelineOptions default (-1) will end-up "
+ + "with the described Spark default.")
+ @Default.Long(-1)
+ Long getCheckpointDurationMillis();
+
+ void setCheckpointDurationMillis(Long durationMillis);
+
+ @Description(
+ "If set bundleSize will be used for splitting BoundedSources, otherwise default to "
+ + "splitting BoundedSources on Spark defaultParallelism. Most effective when used with "
+ + "Spark dynamicAllocation.")
+ @Default.Long(0)
+ Long getBundleSize();
+
+ @Experimental
+ void setBundleSize(Long value);
+
+ @Description("Enable/disable sending aggregator values to Spark's metric sinks")
+ @Default.Boolean(true)
+ Boolean getEnableSparkMetricSinks();
+
+ void setEnableSparkMetricSinks(Boolean enableSparkMetricSinks);
+
+
+ /**
+ * List of local files to make available to workers.
+ *
+ * <p>Jars are placed on the worker's classpath.
+ *
+ * <p>The default value is the list of jars from the main program's classpath.
+ */
+ @Description(
+ "Jar-Files to send to all workers and put on the classpath. "
+ + "The default value is all files from the classpath.")
+ List<String> getFilesToStage();
+
+ void setFilesToStage(List<String> value);
+}
[beam] 09/20: Organise methods in PipelineTranslator
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 5cb1add61ccd50b37156e71239288333a71350a6
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Wed Nov 21 11:31:43 2018 +0100
Organise methods in PipelineTranslator
---
.../spark/structuredstreaming/SparkRunner.java | 1 -
.../translation/PipelineTranslator.java | 64 +++++++++++++---------
2 files changed, 38 insertions(+), 27 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
index 3e3b112..ab2215b 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
@@ -99,7 +99,6 @@ public final class SparkRunner extends PipelineRunner<SparkPipelineResult> {
PipelineTranslator.replaceTransforms(pipeline, options);
PipelineTranslator.prepareFilesToStageForRemoteClusterExecution(options);
PipelineTranslator pipelineTranslator = options.isStreaming() ? new StreamingPipelineTranslator() : new BatchPipelineTranslator();
- //init pipelineTranslator with subclass based on mode and env
pipelineTranslator.translate(pipeline);
}
private void executePipeline(Pipeline pipeline) {}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
index db5c354..8eb1fb6 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
@@ -21,6 +21,9 @@ import org.slf4j.LoggerFactory;
public class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
+ // --------------------------------------------------------------------------------------------
+ // Pipeline preparation methods
+ // --------------------------------------------------------------------------------------------
/**
* Local configurations work in the same JVM and have no problems with improperly formatted files
* on classpath (eg. directories with .class files or empty directories). Prepare files for
@@ -49,32 +52,6 @@ public class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
}
}
- /**
- * Utility formatting method.
- *
- * @param n number of spaces to generate
- * @return String with "|" followed by n spaces
- */
- protected static String genSpaces(int n) {
- StringBuilder builder = new StringBuilder();
- for (int i = 0; i < n; i++) {
- builder.append("| ");
- }
- return builder.toString();
- }
-
- /**
- * Translates the pipeline by passing this class as a visitor.
- *
- * @param pipeline The pipeline to be translated
- */
- public void translate(Pipeline pipeline) {
- pipeline.traverseTopologically(this);
- }
-
-
-
-
/** The translation mode of the Beam Pipeline. */
private enum TranslationMode {
@@ -116,4 +93,39 @@ public class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
}
}
+ // --------------------------------------------------------------------------------------------
+ // Pipeline utility methods
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Utility formatting method.
+ *
+ * @param n number of spaces to generate
+ * @return String with "|" followed by n spaces
+ */
+ protected static String genSpaces(int n) {
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < n; i++) {
+ builder.append("| ");
+ }
+ return builder.toString();
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Pipeline visitor methods
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Translates the pipeline by passing this class as a visitor.
+ *
+ * @param pipeline The pipeline to be translated
+ */
+ public void translate(Pipeline pipeline) {
+ pipeline.traverseTopologically(this);
+ }
+
+
+
+
+
}
[beam] 11/20: Refactoring: -move batch/streaming common translation
visitor and utility methods to PipelineTranslator -rename batch dedicated
classes to Batch* to differentiate with their streaming counterparts
-Introduce TranslationContext for common batch/streaming components
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 3ce792dfa5258ee3d28e97eebf9266b81161f186
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Wed Nov 21 16:51:40 2018 +0100
Refactoring:
-move batch/streaming common translation visitor and utility methods to PipelineTranslator
-rename batch dedicated classes to Batch* to differentiate with their streaming counterparts
-Introduce TranslationContext for common batch/streaming components
---
.../translation/PipelineTranslator.java | 73 ++++++++++++++++++-
.../translation/TransformTranslator.java | 11 +++
.../translation/TranslationContext.java | 13 ++++
.../batch/BatchCombinePerKeyTranslator.java | 17 +++++
.../batch/BatchFlattenPCollectionTranslator.java | 16 ++++
.../batch/BatchGroupByKeyTranslator.java | 17 +++++
.../translation/batch/BatchParDoTranslator.java | 16 ++++
.../translation/batch/BatchPipelineTranslator.java | 85 +++-------------------
.../batch/BatchReadSourceTranslator.java | 15 ++++
.../batch/BatchReshuffleTranslator.java | 12 +++
.../batch/BatchTransformTranslator.java | 11 ---
.../translation/batch/BatchTranslationContext.java | 12 +--
.../batch/BatchWindowAssignTranslator.java | 14 ++++
.../batch/CombinePerKeyTranslatorBatch.java | 14 ----
.../batch/FlattenPCollectionTranslatorBatch.java | 13 ----
.../batch/GroupByKeyTranslatorBatch.java | 14 ----
.../translation/batch/ParDoTranslatorBatch.java | 13 ----
.../batch/ReadSourceTranslatorBatch.java | 12 ---
.../batch/ReshuffleTranslatorBatch.java | 11 ---
.../batch/WindowAssignTranslatorBatch.java | 12 ---
.../streaming/StreamingPipelineTranslator.java | 6 ++
.../streaming/StreamingTranslationContext.java | 7 ++
22 files changed, 227 insertions(+), 187 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
index 8eb1fb6..62e87f2 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
@@ -1,5 +1,6 @@
package org.apache.beam.runners.spark.structuredstreaming.translation;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
import org.apache.beam.runners.core.construction.PipelineResources;
import org.apache.beam.runners.spark.SparkTransformOverrides;
import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
@@ -7,11 +8,11 @@ import org.apache.beam.runners.spark.structuredstreaming.translation.batch.Batch
import org.apache.beam.runners.spark.structuredstreaming.translation.streaming.StreamingPipelineTranslator;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
/**
/**
* The role of this class is to detect the pipeline mode and to translate the Beam operators to their Spark counterparts. If we have
@@ -19,7 +20,11 @@ import org.slf4j.LoggerFactory;
* case, i.e. for a batch job, a {@link BatchPipelineTranslator} is created. Correspondingly,
*/
-public class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
+public abstract class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
+ private int depth = 0;
+ private static final Logger LOG = LoggerFactory.getLogger(PipelineTranslator.class);
+ protected TranslationContext translationContext;
+
// --------------------------------------------------------------------------------------------
// Pipeline preparation methods
@@ -103,7 +108,7 @@ public class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
* @param n number of spaces to generate
* @return String with "|" followed by n spaces
*/
- protected static String genSpaces(int n) {
+ private static String genSpaces(int n) {
StringBuilder builder = new StringBuilder();
for (int i = 0; i < n; i++) {
builder.append("| ");
@@ -111,8 +116,31 @@ public class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
return builder.toString();
}
+ /**
+ * get a {@link TransformTranslator} for the given {@link TransformHierarchy.Node}
+ * @param node
+ * @return
+ */
+ protected abstract TransformTranslator<?> getTransformTranslator(TransformHierarchy.Node node);
+
+ private <T extends PTransform<?, ?>> void translateNode(
+ TransformHierarchy.Node node,
+ TransformTranslator<?> transformTranslator) {
+
+ @SuppressWarnings("unchecked")
+ T typedTransform = (T) node.getTransform();
+
+ @SuppressWarnings("unchecked")
+ TransformTranslator<T> typedTransformTranslator = (TransformTranslator<T>) transformTranslator;
+
+ // create the applied PTransform on the translationContext
+ translationContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
+ typedTransformTranslator.translateNode(typedTransform, translationContext);
+ }
+
+
// --------------------------------------------------------------------------------------------
- // Pipeline visitor methods
+ // Pipeline visitor entry point
// --------------------------------------------------------------------------------------------
/**
@@ -121,11 +149,48 @@ public class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
* @param pipeline The pipeline to be translated
*/
public void translate(Pipeline pipeline) {
+ LOG.info("starting translation of the pipeline using {}", getClass().getName());
pipeline.traverseTopologically(this);
}
+ // --------------------------------------------------------------------------------------------
+ // Pipeline Visitor Methods
+ // --------------------------------------------------------------------------------------------
+ @Override
+ public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
+ LOG.info("{} enterCompositeTransform- {}", genSpaces(depth), node.getFullName());
+ depth++;
+ TransformTranslator<?> transformTranslator = getTransformTranslator(node);
+ if (transformTranslator != null) {
+ translateNode(node, transformTranslator);
+ LOG.info("{} translated- {}", genSpaces(depth), node.getFullName());
+ return CompositeBehavior.DO_NOT_ENTER_TRANSFORM;
+ } else {
+ return CompositeBehavior.ENTER_TRANSFORM;
+ }
+ }
+ @Override
+ public void leaveCompositeTransform(TransformHierarchy.Node node) {
+ depth--;
+ LOG.info("{} leaveCompositeTransform- {}", genSpaces(depth), node.getFullName());
+ }
+
+ @Override
+ public void visitPrimitiveTransform(TransformHierarchy.Node node) {
+ LOG.info("{} visitPrimitiveTransform- {}", genSpaces(depth), node.getFullName());
+
+ // get the transformation corresponding to the node we are
+ // currently visiting and translate it into its Spark alternative.
+ TransformTranslator<?> transformTranslator = getTransformTranslator(node);
+ if (transformTranslator == null) {
+ String transformUrn = PTransformTranslation.urnForTransform(node.getTransform());
+ throw new UnsupportedOperationException(
+ "The transform " + transformUrn + " is currently not supported.");
+ }
+ translateNode(node, transformTranslator);
+ }
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
new file mode 100644
index 0000000..51cdd99
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
@@ -0,0 +1,11 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation;
+
+import org.apache.beam.sdk.transforms.PTransform;
+
+public interface TransformTranslator<TransformT extends PTransform> {
+
+ /** A translator of a {@link PTransform}. */
+
+ void translateNode(TransformT transform, TranslationContext context);
+ }
+
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
new file mode 100644
index 0000000..341ed49
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
@@ -0,0 +1,13 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation;
+
+import org.apache.beam.sdk.runners.AppliedPTransform;
+
+public class TranslationContext {
+
+ private AppliedPTransform<?, ?, ?> currentTransform;
+
+ public void setCurrentTransform(AppliedPTransform<?, ?, ?> currentTransform) {
+ this.currentTransform = currentTransform;
+ }
+
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchCombinePerKeyTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchCombinePerKeyTranslator.java
new file mode 100644
index 0000000..c9cae47
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchCombinePerKeyTranslator.java
@@ -0,0 +1,17 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.TranslationContext;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+class BatchCombinePerKeyTranslator<K, InputT, AccumT, OutputT> implements
+ TransformTranslator<PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>>> {
+
+ @Override public void translateNode(
+ PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>> transform,
+ TranslationContext context) {
+
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchFlattenPCollectionTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchFlattenPCollectionTranslator.java
new file mode 100644
index 0000000..77f6fdb
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchFlattenPCollectionTranslator.java
@@ -0,0 +1,16 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.TranslationContext;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+
+class BatchFlattenPCollectionTranslator<T> implements
+ TransformTranslator<PTransform<PCollectionList<T>, PCollection<T>>> {
+
+ @Override public void translateNode(PTransform<PCollectionList<T>, PCollection<T>> transform,
+ TranslationContext context) {
+
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchGroupByKeyTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchGroupByKeyTranslator.java
new file mode 100644
index 0000000..1bd42f5
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchGroupByKeyTranslator.java
@@ -0,0 +1,17 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.TranslationContext;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+class BatchGroupByKeyTranslator<K, InputT> implements
+ TransformTranslator<PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, Iterable<InputT>>>>> {
+
+ @Override public void translateNode(
+ PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, Iterable<InputT>>>> transform,
+ TranslationContext context) {
+
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchParDoTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchParDoTranslator.java
new file mode 100644
index 0000000..cf8c896
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchParDoTranslator.java
@@ -0,0 +1,16 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.TranslationContext;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+
+class BatchParDoTranslator<InputT, OutputT> implements
+ TransformTranslator<PTransform<PCollection<InputT>, PCollectionTuple>> {
+
+ @Override public void translateNode(PTransform<PCollection<InputT>, PCollectionTuple> transform,
+ TranslationContext context) {
+
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
index 1bf660f..ff92d89 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
@@ -6,13 +6,10 @@ import javax.annotation.Nullable;
import org.apache.beam.runners.core.construction.PTransformTranslation;
import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
import org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.spark.SparkConf;
-import org.apache.spark.sql.SparkSession;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/** {@link Pipeline.PipelineVisitor} for executing a {@link Pipeline} as a Spark batch job. */
@@ -23,37 +20,34 @@ public class BatchPipelineTranslator extends PipelineTranslator {
// Transform Translator Registry
// --------------------------------------------------------------------------------------------
- private BatchTranslationContext translationContext;
- private int depth = 0;
-
@SuppressWarnings("rawtypes")
- private static final Map<String, BatchTransformTranslator> TRANSFORM_TRANSLATORS = new HashMap<>();
+ private static final Map<String, TransformTranslator> TRANSFORM_TRANSLATORS = new HashMap<>();
static {
TRANSFORM_TRANSLATORS.put(PTransformTranslation.COMBINE_PER_KEY_TRANSFORM_URN,
- new CombinePerKeyTranslatorBatch());
+ new BatchCombinePerKeyTranslator());
TRANSFORM_TRANSLATORS
- .put(PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN, new GroupByKeyTranslatorBatch());
- TRANSFORM_TRANSLATORS.put(PTransformTranslation.RESHUFFLE_URN, new ReshuffleTranslatorBatch());
+ .put(PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN, new BatchGroupByKeyTranslator());
+ TRANSFORM_TRANSLATORS.put(PTransformTranslation.RESHUFFLE_URN, new BatchReshuffleTranslator());
TRANSFORM_TRANSLATORS
- .put(PTransformTranslation.FLATTEN_TRANSFORM_URN, new FlattenPCollectionTranslatorBatch());
+ .put(PTransformTranslation.FLATTEN_TRANSFORM_URN, new BatchFlattenPCollectionTranslator());
TRANSFORM_TRANSLATORS
- .put(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN, new WindowAssignTranslatorBatch());
+ .put(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN, new BatchWindowAssignTranslator());
- TRANSFORM_TRANSLATORS.put(PTransformTranslation.PAR_DO_TRANSFORM_URN, new ParDoTranslatorBatch());
+ TRANSFORM_TRANSLATORS.put(PTransformTranslation.PAR_DO_TRANSFORM_URN, new BatchParDoTranslator());
- TRANSFORM_TRANSLATORS.put(PTransformTranslation.READ_TRANSFORM_URN, new ReadSourceTranslatorBatch());
+ TRANSFORM_TRANSLATORS.put(PTransformTranslation.READ_TRANSFORM_URN, new BatchReadSourceTranslator());
}
- private static final Logger LOG = LoggerFactory.getLogger(BatchPipelineTranslator.class);
public BatchPipelineTranslator(SparkPipelineOptions options) {
translationContext = new BatchTranslationContext(options);
}
/** Returns a translator for the given node, if it is possible, otherwise null. */
- private static BatchTransformTranslator<?> getTransformTranslator(TransformHierarchy.Node node) {
+ @Override
+ protected TransformTranslator<?> getTransformTranslator(TransformHierarchy.Node node) {
@Nullable PTransform<?, ?> transform = node.getTransform();
// Root of the graph is null
if (transform == null) {
@@ -64,61 +58,4 @@ public class BatchPipelineTranslator extends PipelineTranslator {
}
- // --------------------------------------------------------------------------------------------
- // Pipeline Visitor Methods
- // --------------------------------------------------------------------------------------------
-
- @Override
- public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
- LOG.info("{} enterCompositeTransform- {}", genSpaces(depth), node.getFullName());
- depth++;
-
- BatchTransformTranslator<?> transformTranslator = getTransformTranslator(node);
-
- if (transformTranslator != null) {
- translateNode(node, transformTranslator);
- LOG.info("{} translated- {}", genSpaces(depth), node.getFullName());
- return CompositeBehavior.DO_NOT_ENTER_TRANSFORM;
- } else {
- return CompositeBehavior.ENTER_TRANSFORM;
- }
- }
-
- @Override
- public void leaveCompositeTransform(TransformHierarchy.Node node) {
- depth--;
- LOG.info("{} leaveCompositeTransform- {}", genSpaces(depth), node.getFullName());
- }
-
- @Override
- public void visitPrimitiveTransform(TransformHierarchy.Node node) {
- LOG.info("{} visitPrimitiveTransform- {}", genSpaces(depth), node.getFullName());
-
- // get the transformation corresponding to the node we are
- // currently visiting and translate it into its Spark alternative.
- BatchTransformTranslator<?> transformTranslator = getTransformTranslator(node);
- if (transformTranslator == null) {
- String transformUrn = PTransformTranslation.urnForTransform(node.getTransform());
- throw new UnsupportedOperationException(
- "The transform " + transformUrn + " is currently not supported.");
- }
- translateNode(node, transformTranslator);
- }
-
- private <T extends PTransform<?, ?>> void translateNode(
- TransformHierarchy.Node node,
- BatchTransformTranslator<?> transformTranslator) {
-
- @SuppressWarnings("unchecked")
- T typedTransform = (T) node.getTransform();
-
- @SuppressWarnings("unchecked")
- BatchTransformTranslator<T> typedTransformTranslator = (BatchTransformTranslator<T>) transformTranslator;
-
- // create the applied PTransform on the translationContext
- translationContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
- typedTransformTranslator.translateNode(typedTransform, translationContext);
- }
-
-
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReadSourceTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReadSourceTranslator.java
new file mode 100644
index 0000000..f5f0351
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReadSourceTranslator.java
@@ -0,0 +1,15 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.TranslationContext;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+
+class BatchReadSourceTranslator<T> implements TransformTranslator<PTransform<PBegin, PCollection<T>>> {
+
+ @Override public void translateNode(PTransform<PBegin, PCollection<T>> transform,
+ TranslationContext context) {
+
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReshuffleTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReshuffleTranslator.java
new file mode 100644
index 0000000..5fab1c8
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReshuffleTranslator.java
@@ -0,0 +1,12 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.TranslationContext;
+import org.apache.beam.sdk.transforms.Reshuffle;
+
+class BatchReshuffleTranslator<K, InputT> implements TransformTranslator<Reshuffle<K, InputT>> {
+
+ @Override public void translateNode(Reshuffle<K, InputT> transform, TranslationContext context) {
+
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTransformTranslator.java
deleted file mode 100644
index ab0cf68..0000000
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTransformTranslator.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
-
-import org.apache.beam.sdk.transforms.PTransform;
-
-public interface BatchTransformTranslator<TransformT extends PTransform> {
-
- /** A translator of a {@link PTransform} in batch mode. */
-
- void translateNode(TransformT transform, BatchTranslationContext context);
- }
-
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
index b53aa19..71ef315 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
@@ -3,6 +3,7 @@ package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
import java.util.HashMap;
import java.util.Map;
import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
+import org.apache.beam.runners.spark.structuredstreaming.translation.TranslationContext;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.values.PValue;
import org.apache.spark.SparkConf;
@@ -12,7 +13,7 @@ import org.apache.spark.sql.SparkSession;
/**
* Keeps track of the {@link Dataset} and the step the translation is in.
*/
-public class BatchTranslationContext {
+public class BatchTranslationContext extends TranslationContext {
private final Map<PValue, Dataset<?>> datasets;
/**
@@ -24,9 +25,6 @@ public class BatchTranslationContext {
private SparkSession sparkSession;
private final SparkPipelineOptions options;
- private AppliedPTransform<?, ?, ?> currentTransform;
-
-
public BatchTranslationContext(SparkPipelineOptions options) {
SparkConf sparkConf = new SparkConf();
sparkConf.setMaster(options.getSparkMaster());
@@ -35,7 +33,7 @@ public class BatchTranslationContext {
sparkConf.setJars(options.getFilesToStage().toArray(new String[0]));
}
- SparkSession sparkSession = SparkSession
+ this.sparkSession = SparkSession
.builder()
.config(sparkConf)
.getOrCreate();
@@ -43,8 +41,4 @@ public class BatchTranslationContext {
this.datasets = new HashMap<>();
this.danglingDataSets = new HashMap<>();
}
-
- public void setCurrentTransform(AppliedPTransform<?, ?, ?> currentTransform) {
- this.currentTransform = currentTransform;
- }
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchWindowAssignTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchWindowAssignTranslator.java
new file mode 100644
index 0000000..fbbced5
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchWindowAssignTranslator.java
@@ -0,0 +1,14 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.TranslationContext;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
+
+class BatchWindowAssignTranslator<T> implements
+ TransformTranslator<PTransform<PCollection<T>, PCollection<T>>> {
+
+ @Override public void translateNode(PTransform<PCollection<T>, PCollection<T>> transform,
+ TranslationContext context) {
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
deleted file mode 100644
index 6099fbc..0000000
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
+++ /dev/null
@@ -1,14 +0,0 @@
-package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
-
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-
-class CombinePerKeyTranslatorBatch<K, InputT, AccumT, OutputT> implements BatchTransformTranslator<PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>>> {
-
- @Override public void translateNode(
- PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>> transform,
- BatchTranslationContext context) {
-
- }
-}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenPCollectionTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenPCollectionTranslatorBatch.java
deleted file mode 100644
index 281eda9..0000000
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenPCollectionTranslatorBatch.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
-
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionList;
-
-class FlattenPCollectionTranslatorBatch<T> implements BatchTransformTranslator<PTransform<PCollectionList<T>, PCollection<T>>> {
-
- @Override public void translateNode(PTransform<PCollectionList<T>, PCollection<T>> transform,
- BatchTranslationContext context) {
-
- }
-}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java
deleted file mode 100644
index bb0ccc1..0000000
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java
+++ /dev/null
@@ -1,14 +0,0 @@
-package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
-
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-
-class GroupByKeyTranslatorBatch<K, InputT> implements BatchTransformTranslator<PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, Iterable<InputT>>>>> {
-
- @Override public void translateNode(
- PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, Iterable<InputT>>>> transform,
- BatchTranslationContext context) {
-
- }
-}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java
deleted file mode 100644
index 4477853..0000000
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
-
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionTuple;
-
-class ParDoTranslatorBatch<InputT, OutputT> implements BatchTransformTranslator<PTransform<PCollection<InputT>, PCollectionTuple>> {
-
- @Override public void translateNode(PTransform<PCollection<InputT>, PCollectionTuple> transform,
- BatchTranslationContext context) {
-
- }
-}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java
deleted file mode 100644
index a30fa70..0000000
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
-
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PBegin;
-import org.apache.beam.sdk.values.PCollection;
-
-class ReadSourceTranslatorBatch<T> implements BatchTransformTranslator<PTransform<PBegin, PCollection<T>>> {
-
- @Override public void translateNode(PTransform<PBegin, PCollection<T>> transform, BatchTranslationContext context) {
-
- }
-}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java
deleted file mode 100644
index 6283fdb..0000000
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
-
-import org.apache.beam.sdk.transforms.Reshuffle;
-
-class ReshuffleTranslatorBatch<K, InputT> implements BatchTransformTranslator<Reshuffle<K, InputT>> {
-
- @Override public void translateNode(Reshuffle<K, InputT> transform,
- BatchTranslationContext context) {
-
- }
-}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java
deleted file mode 100644
index 21b71b9..0000000
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
-
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.values.PCollection;
-
-class WindowAssignTranslatorBatch<T> implements BatchTransformTranslator<PTransform<PCollection<T>, PCollection<T>>> {
-
- @Override public void translateNode(PTransform<PCollection<T>, PCollection<T>> transform,
- BatchTranslationContext context) {
-
- }
-}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
index 7bed930..9303d59 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
@@ -2,9 +2,15 @@ package org.apache.beam.runners.spark.structuredstreaming.translation.streaming;
import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
import org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
+import org.apache.beam.sdk.runners.TransformHierarchy;
public class StreamingPipelineTranslator extends PipelineTranslator {
public StreamingPipelineTranslator(SparkPipelineOptions options) {
}
+
+ @Override protected TransformTranslator<?> getTransformTranslator(TransformHierarchy.Node node) {
+ return null;
+ }
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java
new file mode 100644
index 0000000..460dbf6
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java
@@ -0,0 +1,7 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.streaming;
+
+import org.apache.beam.runners.spark.structuredstreaming.translation.TranslationContext;
+
+public class StreamingTranslationContext extends TranslationContext {
+
+}
[beam] 16/20: apply spotless for e-formatting
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 92ed1300cf123706dab27665a49b49441a6c8deb
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Thu Nov 22 12:04:11 2018 +0100
apply spotless for e-formatting
---
.../structuredstreaming/SparkPipelineOptions.java | 1 -
.../structuredstreaming/SparkPipelineResult.java | 32 +++++++++++---
.../spark/structuredstreaming/SparkRunner.java | 41 +++++++++++++----
.../translation/PipelineTranslator.java | 46 ++++++++++++-------
.../translation/TransformTranslator.java | 21 +++++++--
.../translation/TranslationContext.java | 27 +++++++++---
.../batch/BatchCombinePerKeyTranslator.java | 29 +++++++++---
.../batch/BatchFlattenPCollectionTranslator.java | 28 +++++++++---
.../batch/BatchGroupByKeyTranslator.java | 29 +++++++++---
.../translation/batch/BatchParDoTranslator.java | 28 +++++++++---
.../translation/batch/BatchPipelineTranslator.java | 51 ++++++++++++++--------
.../batch/BatchReadSourceTranslator.java | 27 +++++++++---
.../batch/BatchReshuffleTranslator.java | 22 ++++++++--
.../translation/batch/BatchTranslationContext.java | 24 +++++++---
.../batch/BatchWindowAssignTranslator.java | 27 +++++++++---
.../streaming/StreamingPipelineTranslator.java | 32 +++++++++++---
.../streaming/StreamingTranslationContext.java | 21 +++++++--
17 files changed, 375 insertions(+), 111 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineOptions.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineOptions.java
index d381b5f..2e6653b 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineOptions.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineOptions.java
@@ -89,7 +89,6 @@ public interface SparkPipelineOptions
void setEnableSparkMetricSinks(Boolean enableSparkMetricSinks);
-
/**
* List of local files to make available to workers.
*
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java
index 82d1b90..a8b3640 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming;
import java.io.IOException;
@@ -7,23 +24,28 @@ import org.joda.time.Duration;
public class SparkPipelineResult implements PipelineResult {
- @Override public State getState() {
+ @Override
+ public State getState() {
return null;
}
- @Override public State cancel() throws IOException {
+ @Override
+ public State cancel() throws IOException {
return null;
}
- @Override public State waitUntilFinish(Duration duration) {
+ @Override
+ public State waitUntilFinish(Duration duration) {
return null;
}
- @Override public State waitUntilFinish() {
+ @Override
+ public State waitUntilFinish() {
return null;
}
- @Override public MetricResults metrics() {
+ @Override
+ public MetricResults metrics() {
return null;
}
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
index de20133..3a530f0 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
@@ -1,9 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming;
import static org.apache.beam.runners.core.construction.PipelineResources.detectClassPathResourcesToStage;
-import org.apache.beam.runners.spark.structuredstreaming.translation.batch.BatchPipelineTranslator;
import org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.batch.BatchPipelineTranslator;
import org.apache.beam.runners.spark.structuredstreaming.translation.streaming.StreamingPipelineTranslator;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineRunner;
@@ -65,12 +82,14 @@ public final class SparkRunner extends PipelineRunner<SparkPipelineResult> {
* @return A pipeline runner that will execute with specified options.
*/
public static SparkRunner fromOptions(PipelineOptions options) {
- SparkPipelineOptions sparkOptions = PipelineOptionsValidator
- .validate(SparkPipelineOptions.class, options);
+ SparkPipelineOptions sparkOptions =
+ PipelineOptionsValidator.validate(SparkPipelineOptions.class, options);
if (sparkOptions.getFilesToStage() == null) {
- sparkOptions.setFilesToStage(detectClassPathResourcesToStage(SparkRunner.class.getClassLoader()));
- LOG.info("PipelineOptions.filesToStage was not specified. "
+ sparkOptions.setFilesToStage(
+ detectClassPathResourcesToStage(SparkRunner.class.getClassLoader()));
+ LOG.info(
+ "PipelineOptions.filesToStage was not specified. "
+ "Defaulting to files from the classpath: will stage {} files. "
+ "Enable logging at DEBUG level to see which files will be staged.",
sparkOptions.getFilesToStage().size());
@@ -88,19 +107,23 @@ public final class SparkRunner extends PipelineRunner<SparkPipelineResult> {
this.options = options;
}
- @Override public SparkPipelineResult run(final Pipeline pipeline) {
+ @Override
+ public SparkPipelineResult run(final Pipeline pipeline) {
translatePipeline(pipeline);
executePipeline(pipeline);
return new SparkPipelineResult();
}
- private void translatePipeline(Pipeline pipeline){
+ private void translatePipeline(Pipeline pipeline) {
PipelineTranslator.detectTranslationMode(pipeline, options);
PipelineTranslator.replaceTransforms(pipeline, options);
PipelineTranslator.prepareFilesToStageForRemoteClusterExecution(options);
- PipelineTranslator pipelineTranslator = options.isStreaming() ? new StreamingPipelineTranslator(options) : new BatchPipelineTranslator(options);
+ PipelineTranslator pipelineTranslator =
+ options.isStreaming()
+ ? new StreamingPipelineTranslator(options)
+ : new BatchPipelineTranslator(options);
pipelineTranslator.translate(pipeline);
}
- private void executePipeline(Pipeline pipeline) {}
+ private void executePipeline(Pipeline pipeline) {}
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
index c05fc92..bb40631 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming.translation;
import org.apache.beam.runners.core.construction.PTransformTranslation;
@@ -14,17 +31,16 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* {@link Pipeline.PipelineVisitor} that translates the Beam operators to their Spark counterparts.
- * It also does the pipeline preparation: mode detection, transforms replacement, classpath preparation.
- * If we have a streaming job, it is instantiated as a {@link StreamingPipelineTranslator}.
- * If we have a batch job, it is instantiated as a {@link BatchPipelineTranslator}.
+ * It also does the pipeline preparation: mode detection, transforms replacement, classpath
+ * preparation. If we have a streaming job, it is instantiated as a {@link
+ * StreamingPipelineTranslator}. If we have a batch job, it is instantiated as a {@link
+ * BatchPipelineTranslator}.
*/
-
-public abstract class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
+public abstract class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults {
private int depth = 0;
private static final Logger LOG = LoggerFactory.getLogger(PipelineTranslator.class);
protected TranslationContext translationContext;
-
// --------------------------------------------------------------------------------------------
// Pipeline preparation methods
// --------------------------------------------------------------------------------------------
@@ -41,13 +57,14 @@ public abstract class PipelineTranslator extends Pipeline.PipelineVisitor.Defaul
}
}
- public static void replaceTransforms(Pipeline pipeline, SparkPipelineOptions options){
+ public static void replaceTransforms(Pipeline pipeline, SparkPipelineOptions options) {
pipeline.replaceAll(SparkTransformOverrides.getDefaultOverrides(options.isStreaming()));
-
}
-
- /** Visit the pipeline to determine the translation mode (batch/streaming) and update options accordingly. */
+ /**
+ * Visit the pipeline to determine the translation mode (batch/streaming) and update options
+ * accordingly.
+ */
public static void detectTranslationMode(Pipeline pipeline, SparkPipelineOptions options) {
TranslationModeDetector detector = new TranslationModeDetector();
pipeline.traverseTopologically(detector);
@@ -117,17 +134,15 @@ public abstract class PipelineTranslator extends Pipeline.PipelineVisitor.Defaul
/**
* get a {@link TransformTranslator} for the given {@link TransformHierarchy.Node}
+ *
* @param node
* @return
*/
protected abstract TransformTranslator<?> getTransformTranslator(TransformHierarchy.Node node);
- /**
- * Apply the given TransformTranslator to the given node.
- */
+ /** Apply the given TransformTranslator to the given node. */
private <T extends PTransform<?, ?>> void applyTransformTranslator(
- TransformHierarchy.Node node,
- TransformTranslator<?> transformTranslator) {
+ TransformHierarchy.Node node, TransformTranslator<?> transformTranslator) {
// create the applied PTransform on the translationContext
translationContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
@@ -141,7 +156,6 @@ public abstract class PipelineTranslator extends Pipeline.PipelineVisitor.Defaul
typedTransformTranslator.translateTransform(typedTransform, translationContext);
}
-
// --------------------------------------------------------------------------------------------
// Pipeline visitor entry point
// --------------------------------------------------------------------------------------------
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
index 54b0a85..fc55a9e 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming.translation;
import org.apache.beam.sdk.transforms.PTransform;
@@ -5,7 +22,5 @@ import org.apache.beam.sdk.transforms.PTransform;
public interface TransformTranslator<TransformT extends PTransform> {
/** Base class for translators of {@link PTransform}. */
-
void translateTransform(TransformT transform, TranslationContext context);
- }
-
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
index e651e70..8f61d0c 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming.translation;
import java.util.HashMap;
@@ -11,8 +28,8 @@ import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SparkSession;
/**
- * Base class that gives a context for {@link PTransform} translation: keeping track of the datasets,
- * the {@link SparkSession}, the current transform being translated.
+ * Base class that gives a context for {@link PTransform} translation: keeping track of the
+ * datasets, the {@link SparkSession}, the current transform being translated.
*/
public class TranslationContext {
@@ -33,12 +50,8 @@ public class TranslationContext {
sparkConf.setJars(options.getFilesToStage().toArray(new String[0]));
}
- this.sparkSession = SparkSession
- .builder()
- .config(sparkConf)
- .getOrCreate();
+ this.sparkSession = SparkSession.builder().config(sparkConf).getOrCreate();
this.options = options;
this.datasets = new HashMap<>();
}
-
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchCombinePerKeyTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchCombinePerKeyTranslator.java
index 858df18..4a10329 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchCombinePerKeyTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchCombinePerKeyTranslator.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
@@ -6,12 +23,12 @@ import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
-class BatchCombinePerKeyTranslator<K, InputT, AccumT, OutputT> implements
- TransformTranslator<PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>>> {
+class BatchCombinePerKeyTranslator<K, InputT, AccumT, OutputT>
+ implements TransformTranslator<
+ PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>>> {
- @Override public void translateTransform(
+ @Override
+ public void translateTransform(
PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>> transform,
- TranslationContext context) {
-
- }
+ TranslationContext context) {}
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchFlattenPCollectionTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchFlattenPCollectionTranslator.java
index 90c487a..d24f60c 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchFlattenPCollectionTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchFlattenPCollectionTranslator.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
@@ -6,11 +23,10 @@ import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionList;
-class BatchFlattenPCollectionTranslator<T> implements
- TransformTranslator<PTransform<PCollectionList<T>, PCollection<T>>> {
+class BatchFlattenPCollectionTranslator<T>
+ implements TransformTranslator<PTransform<PCollectionList<T>, PCollection<T>>> {
- @Override public void translateTransform(PTransform<PCollectionList<T>, PCollection<T>> transform,
- TranslationContext context) {
-
- }
+ @Override
+ public void translateTransform(
+ PTransform<PCollectionList<T>, PCollection<T>> transform, TranslationContext context) {}
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchGroupByKeyTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchGroupByKeyTranslator.java
index 52a3c39..829ba8a 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchGroupByKeyTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchGroupByKeyTranslator.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
@@ -6,12 +23,12 @@ import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
-class BatchGroupByKeyTranslator<K, InputT> implements
- TransformTranslator<PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, Iterable<InputT>>>>> {
+class BatchGroupByKeyTranslator<K, InputT>
+ implements TransformTranslator<
+ PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, Iterable<InputT>>>>> {
- @Override public void translateTransform(
+ @Override
+ public void translateTransform(
PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, Iterable<InputT>>>> transform,
- TranslationContext context) {
-
- }
+ TranslationContext context) {}
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchParDoTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchParDoTranslator.java
index 6e7f342..56aa504 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchParDoTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchParDoTranslator.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
@@ -6,11 +23,10 @@ import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionTuple;
-class BatchParDoTranslator<InputT, OutputT> implements
- TransformTranslator<PTransform<PCollection<InputT>, PCollectionTuple>> {
+class BatchParDoTranslator<InputT, OutputT>
+ implements TransformTranslator<PTransform<PCollection<InputT>, PCollectionTuple>> {
- @Override public void translateTransform(PTransform<PCollection<InputT>, PCollectionTuple> transform,
- TranslationContext context) {
-
- }
+ @Override
+ public void translateTransform(
+ PTransform<PCollection<InputT>, PCollectionTuple> transform, TranslationContext context) {}
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
index 38324c0..6648539 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
import java.util.HashMap;
@@ -11,13 +28,13 @@ import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.transforms.PTransform;
-/** {@link PipelineTranslator} for executing a {@link Pipeline} in Spark in batch mode.
- * This contains only the components specific to batch: {@link BatchTranslationContext},
- * registry of batch {@link TransformTranslator} and registry lookup code. */
-
+/**
+ * {@link PipelineTranslator} for executing a {@link Pipeline} in Spark in batch mode. This contains
+ * only the components specific to batch: {@link BatchTranslationContext}, registry of batch {@link
+ * TransformTranslator} and registry lookup code.
+ */
public class BatchPipelineTranslator extends PipelineTranslator {
-
// --------------------------------------------------------------------------------------------
// Transform Translator Registry
// --------------------------------------------------------------------------------------------
@@ -26,21 +43,23 @@ public class BatchPipelineTranslator extends PipelineTranslator {
private static final Map<String, TransformTranslator> TRANSFORM_TRANSLATORS = new HashMap<>();
static {
- TRANSFORM_TRANSLATORS.put(PTransformTranslation.COMBINE_PER_KEY_TRANSFORM_URN,
- new BatchCombinePerKeyTranslator());
- TRANSFORM_TRANSLATORS
- .put(PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN, new BatchGroupByKeyTranslator());
+ TRANSFORM_TRANSLATORS.put(
+ PTransformTranslation.COMBINE_PER_KEY_TRANSFORM_URN, new BatchCombinePerKeyTranslator());
+ TRANSFORM_TRANSLATORS.put(
+ PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN, new BatchGroupByKeyTranslator());
TRANSFORM_TRANSLATORS.put(PTransformTranslation.RESHUFFLE_URN, new BatchReshuffleTranslator());
- TRANSFORM_TRANSLATORS
- .put(PTransformTranslation.FLATTEN_TRANSFORM_URN, new BatchFlattenPCollectionTranslator());
+ TRANSFORM_TRANSLATORS.put(
+ PTransformTranslation.FLATTEN_TRANSFORM_URN, new BatchFlattenPCollectionTranslator());
- TRANSFORM_TRANSLATORS
- .put(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN, new BatchWindowAssignTranslator());
+ TRANSFORM_TRANSLATORS.put(
+ PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN, new BatchWindowAssignTranslator());
- TRANSFORM_TRANSLATORS.put(PTransformTranslation.PAR_DO_TRANSFORM_URN, new BatchParDoTranslator());
+ TRANSFORM_TRANSLATORS.put(
+ PTransformTranslation.PAR_DO_TRANSFORM_URN, new BatchParDoTranslator());
- TRANSFORM_TRANSLATORS.put(PTransformTranslation.READ_TRANSFORM_URN, new BatchReadSourceTranslator());
+ TRANSFORM_TRANSLATORS.put(
+ PTransformTranslation.READ_TRANSFORM_URN, new BatchReadSourceTranslator());
}
public BatchPipelineTranslator(SparkPipelineOptions options) {
@@ -58,6 +77,4 @@ public class BatchPipelineTranslator extends PipelineTranslator {
@Nullable String urn = PTransformTranslation.urnForTransformOrNull(transform);
return (urn == null) ? null : TRANSFORM_TRANSLATORS.get(urn);
}
-
-
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReadSourceTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReadSourceTranslator.java
index 4236b1c..d9fcfbb 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReadSourceTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReadSourceTranslator.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
@@ -6,10 +23,10 @@ import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
-class BatchReadSourceTranslator<T> implements TransformTranslator<PTransform<PBegin, PCollection<T>>> {
+class BatchReadSourceTranslator<T>
+ implements TransformTranslator<PTransform<PBegin, PCollection<T>>> {
- @Override public void translateTransform(PTransform<PBegin, PCollection<T>> transform,
- TranslationContext context) {
-
- }
+ @Override
+ public void translateTransform(
+ PTransform<PBegin, PCollection<T>> transform, TranslationContext context) {}
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReshuffleTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReshuffleTranslator.java
index 5baa331..1423308 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReshuffleTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReshuffleTranslator.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
@@ -6,7 +23,6 @@ import org.apache.beam.sdk.transforms.Reshuffle;
class BatchReshuffleTranslator<K, InputT> implements TransformTranslator<Reshuffle<K, InputT>> {
- @Override public void translateTransform(Reshuffle<K, InputT> transform, TranslationContext context) {
-
- }
+ @Override
+ public void translateTransform(Reshuffle<K, InputT> transform, TranslationContext context) {}
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
index 02aad71..6f50895 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
@@ -1,18 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
import java.util.HashMap;
import java.util.Map;
import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
import org.apache.beam.runners.spark.structuredstreaming.translation.TranslationContext;
-import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.values.PValue;
-import org.apache.spark.SparkConf;
import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.SparkSession;
-/**
- * This class contains only batch specific context components.
- */
+/** This class contains only batch specific context components. */
public class BatchTranslationContext extends TranslationContext {
/**
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchWindowAssignTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchWindowAssignTranslator.java
index 1a8f68b..65a7cae 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchWindowAssignTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchWindowAssignTranslator.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator;
@@ -5,10 +22,10 @@ import org.apache.beam.runners.spark.structuredstreaming.translation.Translation
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PCollection;
-class BatchWindowAssignTranslator<T> implements
- TransformTranslator<PTransform<PCollection<T>, PCollection<T>>> {
+class BatchWindowAssignTranslator<T>
+ implements TransformTranslator<PTransform<PCollection<T>, PCollection<T>>> {
- @Override public void translateTransform(PTransform<PCollection<T>, PCollection<T>> transform,
- TranslationContext context) {
- }
+ @Override
+ public void translateTransform(
+ PTransform<PCollection<T>, PCollection<T>> transform, TranslationContext context) {}
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
index 9cbfbed..437aa25 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming.translation.streaming;
import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
@@ -6,16 +23,17 @@ import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTr
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.TransformHierarchy;
-/** {@link PipelineTranslator} for executing a {@link Pipeline} in Spark in streaming mode.
- * This contains only the components specific to streaming: {@link StreamingTranslationContext},
- * registry of batch {@link TransformTranslator} and registry lookup code. */
-
+/**
+ * {@link PipelineTranslator} for executing a {@link Pipeline} in Spark in streaming mode. This
+ * contains only the components specific to streaming: {@link StreamingTranslationContext}, registry
+ * of batch {@link TransformTranslator} and registry lookup code.
+ */
public class StreamingPipelineTranslator extends PipelineTranslator {
- public StreamingPipelineTranslator(SparkPipelineOptions options) {
- }
+ public StreamingPipelineTranslator(SparkPipelineOptions options) {}
- @Override protected TransformTranslator<?> getTransformTranslator(TransformHierarchy.Node node) {
+ @Override
+ protected TransformTranslator<?> getTransformTranslator(TransformHierarchy.Node node) {
return null;
}
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java
index ebccfa7..f827cc4 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java
@@ -1,11 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.beam.runners.spark.structuredstreaming.translation.streaming;
import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
import org.apache.beam.runners.spark.structuredstreaming.translation.TranslationContext;
-/**
- * This class contains only streaming specific context components.
- */
+/** This class contains only streaming specific context components. */
public class StreamingTranslationContext extends TranslationContext {
public StreamingTranslationContext(SparkPipelineOptions options) {
[beam] 04/20: Start pipeline translation
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit ad3230463eac22179cd50eade618d9c2879caf76
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Fri Nov 16 19:06:04 2018 +0100
Start pipeline translation
---
.../structuredstreaming/SparkPipelineResult.java | 29 +++
.../spark/structuredstreaming/SparkRunner.java | 108 +++++++++
.../translation/BatchPipelineTranslator.java | 20 ++
.../translation/EvaluationContext.java | 261 ---------------------
.../translation/PipelineTranslator.java | 94 ++++++++
.../translation/SparkTransformOverrides.java | 52 ++++
.../translation/StreamingPipelineTranslator.java | 5 +
7 files changed, 308 insertions(+), 261 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java
new file mode 100644
index 0000000..82d1b90
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java
@@ -0,0 +1,29 @@
+package org.apache.beam.runners.spark.structuredstreaming;
+
+import java.io.IOException;
+import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.metrics.MetricResults;
+import org.joda.time.Duration;
+
+public class SparkPipelineResult implements PipelineResult {
+
+ @Override public State getState() {
+ return null;
+ }
+
+ @Override public State cancel() throws IOException {
+ return null;
+ }
+
+ @Override public State waitUntilFinish(Duration duration) {
+ return null;
+ }
+
+ @Override public State waitUntilFinish() {
+ return null;
+ }
+
+ @Override public MetricResults metrics() {
+ return null;
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
new file mode 100644
index 0000000..62cd7d3
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
@@ -0,0 +1,108 @@
+package org.apache.beam.runners.spark.structuredstreaming;
+
+import static org.apache.beam.runners.core.construction.PipelineResources.detectClassPathResourcesToStage;
+
+import org.apache.beam.runners.spark.structuredstreaming.translation.BatchPipelineTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.StreamingPipelineTranslator;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineRunner;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.options.PipelineOptionsValidator;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The SparkRunner translate operations defined on a pipeline to a representation executable by
+ * Spark, and then submitting the job to Spark to be executed. If we wanted to run a Beam pipeline
+ * with the default options of a single threaded spark instance in local mode, we would do the
+ * following:
+ *
+ * <p>{@code Pipeline p = [logic for pipeline creation] SparkPipelineResult result =
+ * (SparkPipelineResult) p.run(); }
+ *
+ * <p>To create a pipeline runner to run against a different spark cluster, with a custom master url
+ * we would do the following:
+ *
+ * <p>{@code Pipeline p = [logic for pipeline creation] SparkPipelineOptions options =
+ * SparkPipelineOptionsFactory.create(); options.setSparkMaster("spark://host:port");
+ * SparkPipelineResult result = (SparkPipelineResult) p.run(); }
+ */
+public final class SparkRunner extends PipelineRunner<SparkPipelineResult> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SparkRunner.class);
+
+ /** Options used in this pipeline runner. */
+ private final SparkPipelineOptions options;
+
+ /**
+ * Creates and returns a new SparkRunner with default options. In particular, against a spark
+ * instance running in local mode.
+ *
+ * @return A pipeline runner with default options.
+ */
+ public static SparkRunner create() {
+ SparkPipelineOptions options = PipelineOptionsFactory.as(SparkPipelineOptions.class);
+ options.setRunner(SparkRunner.class);
+ return new SparkRunner(options);
+ }
+
+ /**
+ * Creates and returns a new SparkRunner with specified options.
+ *
+ * @param options The SparkPipelineOptions to use when executing the job.
+ * @return A pipeline runner that will execute with specified options.
+ */
+ public static SparkRunner create(SparkPipelineOptions options) {
+ return new SparkRunner(options);
+ }
+
+ /**
+ * Creates and returns a new SparkRunner with specified options.
+ *
+ * @param options The PipelineOptions to use when executing the job.
+ * @return A pipeline runner that will execute with specified options.
+ */
+ public static SparkRunner fromOptions(PipelineOptions options) {
+ SparkPipelineOptions sparkOptions = PipelineOptionsValidator
+ .validate(SparkPipelineOptions.class, options);
+
+ if (sparkOptions.getFilesToStage() == null) {
+ sparkOptions.setFilesToStage(detectClassPathResourcesToStage(SparkRunner.class.getClassLoader()));
+ LOG.info("PipelineOptions.filesToStage was not specified. "
+ + "Defaulting to files from the classpath: will stage {} files. "
+ + "Enable logging at DEBUG level to see which files will be staged.",
+ sparkOptions.getFilesToStage().size());
+ LOG.debug("Classpath elements: {}", sparkOptions.getFilesToStage());
+ }
+
+ return new SparkRunner(sparkOptions);
+ }
+
+ /**
+ * No parameter constructor defaults to running this pipeline in Spark's local mode, in a single
+ * thread.
+ */
+ private SparkRunner(SparkPipelineOptions options) {
+ this.options = options;
+ }
+
+ @Override public SparkPipelineResult run(final Pipeline pipeline) {
+ translatePipeline(pipeline);
+ executePipeline(pipeline);
+ return new SparkPipelineResult();
+ }
+
+ private void translatePipeline(Pipeline pipeline){
+ PipelineTranslator.detectTranslationMode(pipeline, options);
+ PipelineTranslator.replaceTransforms(pipeline, options);
+ PipelineTranslator.prepareFilesToStageForRemoteClusterExecution(options);
+ PipelineTranslator translator = options.isStreaming() ? new StreamingPipelineTranslator() : new BatchPipelineTranslator()
+ //init translator with subclass based on mode and env
+ translator.translate(pipeline);
+ }
+ public void executePipeline(Pipeline pipeline) {}
+
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/BatchPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/BatchPipelineTranslator.java
new file mode 100644
index 0000000..e66555c
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/BatchPipelineTranslator.java
@@ -0,0 +1,20 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.values.PValue;
+
+public class BatchPipelineTranslator extends PipelineTranslator {
+
+
+ @Override public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
+ return super.enterCompositeTransform(node);
+ }
+
+
+ @Override public void visitPrimitiveTransform(TransformHierarchy.Node node) {
+ super.visitPrimitiveTransform(node);
+ }
+
+
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/EvaluationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/EvaluationContext.java
deleted file mode 100644
index 47a3098..0000000
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/EvaluationContext.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.runners.spark.structuredstreaming.translation;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import com.google.common.collect.Iterables;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
-import org.apache.beam.runners.core.construction.TransformInputs;
-import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.runners.AppliedPTransform;
-import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.PValue;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SparkSession;
-import org.apache.spark.streaming.api.java.JavaStreamingContext;
-
-/**
- * The EvaluationContext allows us to define pipeline instructions and translate between {@code
- * PObject<T>}s or {@code PCollection<T>}s and Ts or DStreams/RDDs of Ts.
- */
-public class EvaluationContext {
- private SparkSession sparkSession;
- private final Pipeline pipeline;
- private final Map<PValue, Dataset> datasets = new LinkedHashMap<>();
- private final Map<PValue, Dataset> pcollections = new LinkedHashMap<>();
- private final Set<Dataset> leaves = new LinkedHashSet<>();
- private final Map<PValue, Object> pobjects = new LinkedHashMap<>();
- private AppliedPTransform<?, ?, ?> currentTransform;
- private final SparkPCollectionView pviews = new SparkPCollectionView();
- private final Map<PCollection, Long> cacheCandidates = new HashMap<>();
- private final PipelineOptions options;
- private final SerializablePipelineOptions serializableOptions;
-
- public EvaluationContext(JavaSparkContext jsc, Pipeline pipeline, PipelineOptions options) {
- this.jsc = jsc;
- this.pipeline = pipeline;
- this.options = options;
- this.serializableOptions = new SerializablePipelineOptions(options);
- }
-
- public EvaluationContext(
- JavaSparkContext jsc, Pipeline pipeline, PipelineOptions options, JavaStreamingContext jssc) {
- this(jsc, pipeline, options);
- this.jssc = jssc;
- }
-
- public JavaSparkContext getSparkContext() {
- return jsc;
- }
-
- public JavaStreamingContext getStreamingContext() {
- return jssc;
- }
-
- public Pipeline getPipeline() {
- return pipeline;
- }
-
- public PipelineOptions getOptions() {
- return options;
- }
-
- public SerializablePipelineOptions getSerializableOptions() {
- return serializableOptions;
- }
-
- public void setCurrentTransform(AppliedPTransform<?, ?, ?> transform) {
- this.currentTransform = transform;
- }
-
- public AppliedPTransform<?, ?, ?> getCurrentTransform() {
- return currentTransform;
- }
-
- public <T extends PValue> T getInput(PTransform<T, ?> transform) {
- @SuppressWarnings("unchecked")
- T input =
- (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
- return input;
- }
-
- public <T> Map<TupleTag<?>, PValue> getInputs(PTransform<?, ?> transform) {
- checkArgument(currentTransform != null, "can only be called with non-null currentTransform");
- checkArgument(
- currentTransform.getTransform() == transform, "can only be called with current transform");
- return currentTransform.getInputs();
- }
-
- public <T extends PValue> T getOutput(PTransform<?, T> transform) {
- @SuppressWarnings("unchecked")
- T output = (T) Iterables.getOnlyElement(getOutputs(transform).values());
- return output;
- }
-
- public Map<TupleTag<?>, PValue> getOutputs(PTransform<?, ?> transform) {
- checkArgument(currentTransform != null, "can only be called with non-null currentTransform");
- checkArgument(
- currentTransform.getTransform() == transform, "can only be called with current transform");
- return currentTransform.getOutputs();
- }
-
- public Map<TupleTag<?>, Coder<?>> getOutputCoders() {
- return currentTransform
- .getOutputs()
- .entrySet()
- .stream()
- .filter(e -> e.getValue() instanceof PCollection)
- .collect(Collectors.toMap(e -> e.getKey(), e -> ((PCollection) e.getValue()).getCoder()));
- }
-
- private boolean shouldCache(PValue pvalue) {
- if ((pvalue instanceof PCollection)
- && cacheCandidates.containsKey(pvalue)
- && cacheCandidates.get(pvalue) > 1) {
- return true;
- }
- return false;
- }
-
- public void putDataset(
- PTransform<?, ? extends PValue> transform, Dataset dataset, boolean forceCache) {
- putDataset(getOutput(transform), dataset, forceCache);
- }
-
- public void putDataset(PTransform<?, ? extends PValue> transform, Dataset dataset) {
- putDataset(transform, dataset, false);
- }
-
- public void putDataset(PValue pvalue, Dataset dataset, boolean forceCache) {
- try {
- dataset.setName(pvalue.getName());
- } catch (IllegalStateException e) {
- // name not set, ignore
- }
- if ((forceCache || shouldCache(pvalue)) && pvalue instanceof PCollection) {
- // we cache only PCollection
- Coder<?> coder = ((PCollection<?>) pvalue).getCoder();
- Coder<? extends BoundedWindow> wCoder =
- ((PCollection<?>) pvalue).getWindowingStrategy().getWindowFn().windowCoder();
- dataset.cache(storageLevel(), WindowedValue.getFullCoder(coder, wCoder));
- }
- datasets.put(pvalue, dataset);
- leaves.add(dataset);
- }
-
- public Dataset borrowDataset(PTransform<? extends PValue, ?> transform) {
- return borrowDataset(getInput(transform));
- }
-
- public Dataset borrowDataset(PValue pvalue) {
- Dataset dataset = datasets.get(pvalue);
- leaves.remove(dataset);
- return dataset;
- }
-
- /**
- * Computes the outputs for all RDDs that are leaves in the DAG and do not have any actions (like
- * saving to a file) registered on them (i.e. they are performed for side effects).
- */
- public void computeOutputs() {
- for (Dataset dataset : leaves) {
- dataset.action(); // force computation.
- }
- }
-
- /**
- * Retrieve an object of Type T associated with the PValue passed in.
- *
- * @param value PValue to retrieve associated data for.
- * @param <T> Type of object to return.
- * @return Native object.
- */
- @SuppressWarnings("TypeParameterUnusedInFormals")
- public <T> T get(PValue value) {
- if (pobjects.containsKey(value)) {
- T result = (T) pobjects.get(value);
- return result;
- }
- if (pcollections.containsKey(value)) {
- JavaRDD<?> rdd = ((BoundedDataset) pcollections.get(value)).getRDD();
- T res = (T) Iterables.getOnlyElement(rdd.collect());
- pobjects.put(value, res);
- return res;
- }
- throw new IllegalStateException("Cannot resolve un-known PObject: " + value);
- }
-
- /**
- * Return the current views creates in the pipeline.
- *
- * @return SparkPCollectionView
- */
- public SparkPCollectionView getPViews() {
- return pviews;
- }
-
- /**
- * Adds/Replaces a view to the current views creates in the pipeline.
- *
- * @param view - Identifier of the view
- * @param value - Actual value of the view
- * @param coder - Coder of the value
- */
- public void putPView(
- PCollectionView<?> view,
- Iterable<WindowedValue<?>> value,
- Coder<Iterable<WindowedValue<?>>> coder) {
- pviews.putPView(view, value, coder);
- }
-
- /**
- * Get the map of cache candidates hold by the evaluation context.
- *
- * @return The current {@link Map} of cache candidates.
- */
- public Map<PCollection, Long> getCacheCandidates() {
- return this.cacheCandidates;
- }
-
- <T> Iterable<WindowedValue<T>> getWindowedValues(PCollection<T> pcollection) {
- @SuppressWarnings("unchecked")
- BoundedDataset<T> boundedDataset = (BoundedDataset<T>) datasets.get(pcollection);
- leaves.remove(boundedDataset);
- return boundedDataset.getValues(pcollection);
- }
-
- public String storageLevel() {
- return serializableOptions.get().as(SparkPipelineOptions.class).getStorageLevel();
- }
-}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
new file mode 100644
index 0000000..f0ce1e5
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
@@ -0,0 +1,94 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation;
+
+import org.apache.beam.runners.core.construction.PipelineResources;
+import org.apache.beam.runners.spark.SparkTransformOverrides;
+import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Does all the translation work: mode detection, nodes translation.
+ */
+
+public class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
+
+
+ /**
+ * Local configurations work in the same JVM and have no problems with improperly formatted files
+ * on classpath (eg. directories with .class files or empty directories). Prepare files for
+ * staging only when using remote cluster (passing the master address explicitly).
+ */
+ public static void prepareFilesToStageForRemoteClusterExecution(SparkPipelineOptions options) {
+ if (!options.getSparkMaster().matches("local\\[?\\d*\\]?")) {
+ options.setFilesToStage(
+ PipelineResources.prepareFilesForStaging(
+ options.getFilesToStage(), options.getTempLocation()));
+ }
+ }
+
+ public static void replaceTransforms(Pipeline pipeline, SparkPipelineOptions options){
+ pipeline.replaceAll(SparkTransformOverrides.getDefaultOverrides(options.isStreaming()));
+
+ }
+
+
+ /** Visit the pipeline to determine the translation mode (batch/streaming) and update options accordingly. */
+ public static void detectTranslationMode(Pipeline pipeline, SparkPipelineOptions options) {
+ TranslationModeDetector detector = new TranslationModeDetector();
+ pipeline.traverseTopologically(detector);
+ if (detector.getTranslationMode().equals(TranslationMode.STREAMING)) {
+ // set streaming mode if it's a streaming pipeline
+ options.setStreaming(true);
+ }
+ }
+
+
+
+
+
+ /** The translation mode of the Beam Pipeline. */
+ private enum TranslationMode {
+
+ /** Uses the batch mode. */
+ BATCH,
+
+ /** Uses the streaming mode. */
+ STREAMING
+ }
+
+ /** Traverses the Pipeline to determine the {@link TranslationMode} for this pipeline. */
+ private static class TranslationModeDetector extends Pipeline.PipelineVisitor.Defaults {
+ private static final Logger LOG = LoggerFactory.getLogger(TranslationModeDetector.class);
+
+ private TranslationMode translationMode;
+
+ TranslationModeDetector(TranslationMode defaultMode) {
+ this.translationMode = defaultMode;
+ }
+
+ TranslationModeDetector() {
+ this(TranslationMode.BATCH);
+ }
+
+ TranslationMode getTranslationMode() {
+ return translationMode;
+ }
+
+ @Override
+ public void visitValue(PValue value, TransformHierarchy.Node producer) {
+ if (translationMode.equals(TranslationMode.BATCH)) {
+ if (value instanceof PCollection
+ && ((PCollection) value).isBounded() == PCollection.IsBounded.UNBOUNDED) {
+ LOG.info(
+ "Found unbounded PCollection {}. Switching to streaming execution.", value.getName());
+ translationMode = TranslationMode.STREAMING;
+ }
+ }
+ }
+ }
+
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java
new file mode 100644
index 0000000..897ac01
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark;
+
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import org.apache.beam.runners.core.construction.PTransformMatchers;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
+import org.apache.beam.runners.core.construction.SplittableParDo;
+import org.apache.beam.runners.core.construction.SplittableParDoNaiveBounded;
+import org.apache.beam.runners.core.construction.UnsupportedOverrideFactory;
+import org.apache.beam.sdk.runners.PTransformOverride;
+import org.apache.beam.sdk.transforms.PTransform;
+
+/** {@link PTransform} overrides for Flink runner. */
+public class SparkTransformOverrides {
+ public static List<PTransformOverride> getDefaultOverrides(boolean streaming) {
+ ImmutableList.Builder<PTransformOverride> builder = ImmutableList.builder();
+ // TODO: [BEAM-5358] Support @RequiresStableInput on Spark runner
+ builder.add(
+ PTransformOverride.of(
+ PTransformMatchers.requiresStableInputParDoMulti(),
+ UnsupportedOverrideFactory.withMessage(
+ "Spark runner currently doesn't support @RequiresStableInput annotation.")));
+ if (!streaming) {
+ builder
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory()))
+ .add(
+ PTransformOverride.of(
+ PTransformMatchers.urnEqualTo(PTransformTranslation.SPLITTABLE_PROCESS_KEYED_URN),
+ new SplittableParDoNaiveBounded.OverrideFactory()));
+ }
+ return builder.build();
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/StreamingPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/StreamingPipelineTranslator.java
new file mode 100644
index 0000000..2058b37
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/StreamingPipelineTranslator.java
@@ -0,0 +1,5 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation;
+
+public class StreamingPipelineTranslator extends PipelineTranslator {
+
+}
[beam] 18/20: Add TODOs
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit af6a350fe778c32ab13b039758b0359fa46f7945
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Thu Nov 22 17:10:25 2018 +0100
Add TODOs
---
.../org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
index 3a530f0..b76a530 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
@@ -110,6 +110,7 @@ public final class SparkRunner extends PipelineRunner<SparkPipelineResult> {
@Override
public SparkPipelineResult run(final Pipeline pipeline) {
translatePipeline(pipeline);
+ //TODO initialise other services: checkpointing, metrics system, listeners, ...
executePipeline(pipeline);
return new SparkPipelineResult();
}
[beam] 08/20: Renames: better differenciate pipeline translator for
transform translator
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit b6b426e99f8f4ce322a6327c93af8918318a1563
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Wed Nov 21 10:52:43 2018 +0100
Renames: better differenciate pipeline translator for transform translator
---
.../spark/structuredstreaming/SparkRunner.java | 6 ++--
.../translation/batch/BatchPipelineTranslator.java | 39 +++++++++++-----------
2 files changed, 22 insertions(+), 23 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
index 59c08f7..3e3b112 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
@@ -98,9 +98,9 @@ public final class SparkRunner extends PipelineRunner<SparkPipelineResult> {
PipelineTranslator.detectTranslationMode(pipeline, options);
PipelineTranslator.replaceTransforms(pipeline, options);
PipelineTranslator.prepareFilesToStageForRemoteClusterExecution(options);
- PipelineTranslator translator = options.isStreaming() ? new StreamingPipelineTranslator() : new BatchPipelineTranslator();
- //init translator with subclass based on mode and env
- translator.translate(pipeline);
+ PipelineTranslator pipelineTranslator = options.isStreaming() ? new StreamingPipelineTranslator() : new BatchPipelineTranslator();
+ //init pipelineTranslator with subclass based on mode and env
+ pipelineTranslator.translate(pipeline);
}
private void executePipeline(Pipeline pipeline) {}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
index e20e4c0..2459372 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
@@ -24,39 +24,38 @@ public class BatchPipelineTranslator extends PipelineTranslator {
private int depth = 0;
@SuppressWarnings("rawtypes")
- private static final Map<String, BatchTransformTranslator>
- TRANSLATORS = new HashMap<>();
+ private static final Map<String, BatchTransformTranslator> TRANSFORM_TRANSLATORS = new HashMap<>();
static {
- TRANSLATORS.put(PTransformTranslation.COMBINE_PER_KEY_TRANSFORM_URN,
+ TRANSFORM_TRANSLATORS.put(PTransformTranslation.COMBINE_PER_KEY_TRANSFORM_URN,
new CombinePerKeyTranslatorBatch());
- TRANSLATORS
+ TRANSFORM_TRANSLATORS
.put(PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN, new GroupByKeyTranslatorBatch());
- TRANSLATORS.put(PTransformTranslation.RESHUFFLE_URN, new ReshuffleTranslatorBatch());
+ TRANSFORM_TRANSLATORS.put(PTransformTranslation.RESHUFFLE_URN, new ReshuffleTranslatorBatch());
- TRANSLATORS
+ TRANSFORM_TRANSLATORS
.put(PTransformTranslation.FLATTEN_TRANSFORM_URN, new FlattenPCollectionTranslatorBatch());
- TRANSLATORS
+ TRANSFORM_TRANSLATORS
.put(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN, new WindowAssignTranslatorBatch());
- TRANSLATORS.put(PTransformTranslation.PAR_DO_TRANSFORM_URN, new ParDoTranslatorBatch());
+ TRANSFORM_TRANSLATORS.put(PTransformTranslation.PAR_DO_TRANSFORM_URN, new ParDoTranslatorBatch());
- TRANSLATORS.put(PTransformTranslation.READ_TRANSFORM_URN, new ReadSourceTranslatorBatch());
+ TRANSFORM_TRANSLATORS.put(PTransformTranslation.READ_TRANSFORM_URN, new ReadSourceTranslatorBatch());
}
private static final Logger LOG = LoggerFactory.getLogger(BatchPipelineTranslator.class);
/** Returns a translator for the given node, if it is possible, otherwise null. */
- private static BatchTransformTranslator<?> getTranslator(TransformHierarchy.Node node) {
+ private static BatchTransformTranslator<?> getTransformTranslator(TransformHierarchy.Node node) {
@Nullable PTransform<?, ?> transform = node.getTransform();
// Root of the graph is null
if (transform == null) {
return null;
}
@Nullable String urn = PTransformTranslation.urnForTransformOrNull(transform);
- return (urn == null) ? null : TRANSLATORS.get(urn);
+ return (urn == null) ? null : TRANSFORM_TRANSLATORS.get(urn);
}
@@ -69,10 +68,10 @@ public class BatchPipelineTranslator extends PipelineTranslator {
LOG.info("{} enterCompositeTransform- {}", genSpaces(depth), node.getFullName());
depth++;
- BatchTransformTranslator<?> translator = getTranslator(node);
+ BatchTransformTranslator<?> transformTranslator = getTransformTranslator(node);
- if (translator != null) {
- translateNode(node, translator);
+ if (transformTranslator != null) {
+ translateNode(node, transformTranslator);
LOG.info("{} translated- {}", genSpaces(depth), node.getFullName());
return CompositeBehavior.DO_NOT_ENTER_TRANSFORM;
} else {
@@ -92,28 +91,28 @@ public class BatchPipelineTranslator extends PipelineTranslator {
// get the transformation corresponding to the node we are
// currently visiting and translate it into its Spark alternative.
- BatchTransformTranslator<?> translator = getTranslator(node);
- if (translator == null) {
+ BatchTransformTranslator<?> transformTranslator = getTransformTranslator(node);
+ if (transformTranslator == null) {
String transformUrn = PTransformTranslation.urnForTransform(node.getTransform());
throw new UnsupportedOperationException(
"The transform " + transformUrn + " is currently not supported.");
}
- translateNode(node, translator);
+ translateNode(node, transformTranslator);
}
private <T extends PTransform<?, ?>> void translateNode(
TransformHierarchy.Node node,
- BatchTransformTranslator<?> translator) {
+ BatchTransformTranslator<?> transformTranslator) {
@SuppressWarnings("unchecked")
T typedTransform = (T) node.getTransform();
@SuppressWarnings("unchecked")
- BatchTransformTranslator<T> typedTranslator = (BatchTransformTranslator<T>) translator;
+ BatchTransformTranslator<T> typedTransformTranslator = (BatchTransformTranslator<T>) transformTranslator;
// create the applied PTransform on the translationContext
translationContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
- typedTranslator.translateNode(typedTransform, translationContext);
+ typedTransformTranslator.translateNode(typedTransform, translationContext);
}
[beam] 02/20: Fix missing dep
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 0a89b75c92b1cfb13dc790d6e05dcd0f747a0883
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Wed Nov 14 14:35:08 2018 +0100
Fix missing dep
---
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
.../translation/EvaluationContext.java | 261 +++++++++++++++++++++
2 files changed, 262 insertions(+), 1 deletion(-)
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 8888cd2..9c3af1c 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -433,7 +433,7 @@ class BeamModulePlugin implements Plugin<Project> {
slf4j_jdk14 : "org.slf4j:slf4j-jdk14:1.7.25",
slf4j_log4j12 : "org.slf4j:slf4j-log4j12:1.7.25",
snappy_java : "org.xerial.snappy:snappy-java:1.1.4",
- spark_sql : "org.apache.spark:spark-core_2.11:$spark_structured_streaming_version",
+ spark_sql : "org.apache.spark:spark-sql_2.11:$spark_structured_streaming_version",
spark_core : "org.apache.spark:spark-core_2.11:$spark_version",
spark_network_common : "org.apache.spark:spark-network-common_2.11:$spark_version",
spark_streaming : "org.apache.spark:spark-streaming_2.11:$spark_version",
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/EvaluationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/EvaluationContext.java
new file mode 100644
index 0000000..47a3098
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/EvaluationContext.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.spark.structuredstreaming.translation;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.Iterables;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
+import org.apache.beam.runners.core.construction.TransformInputs;
+import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.streaming.api.java.JavaStreamingContext;
+
+/**
+ * The EvaluationContext allows us to define pipeline instructions and translate between {@code
+ * PObject<T>}s or {@code PCollection<T>}s and Ts or DStreams/RDDs of Ts.
+ */
+public class EvaluationContext {
+ private SparkSession sparkSession;
+ private final Pipeline pipeline;
+ private final Map<PValue, Dataset> datasets = new LinkedHashMap<>();
+ private final Map<PValue, Dataset> pcollections = new LinkedHashMap<>();
+ private final Set<Dataset> leaves = new LinkedHashSet<>();
+ private final Map<PValue, Object> pobjects = new LinkedHashMap<>();
+ private AppliedPTransform<?, ?, ?> currentTransform;
+ private final SparkPCollectionView pviews = new SparkPCollectionView();
+ private final Map<PCollection, Long> cacheCandidates = new HashMap<>();
+ private final PipelineOptions options;
+ private final SerializablePipelineOptions serializableOptions;
+
+ public EvaluationContext(JavaSparkContext jsc, Pipeline pipeline, PipelineOptions options) {
+ this.jsc = jsc;
+ this.pipeline = pipeline;
+ this.options = options;
+ this.serializableOptions = new SerializablePipelineOptions(options);
+ }
+
+ public EvaluationContext(
+ JavaSparkContext jsc, Pipeline pipeline, PipelineOptions options, JavaStreamingContext jssc) {
+ this(jsc, pipeline, options);
+ this.jssc = jssc;
+ }
+
+ public JavaSparkContext getSparkContext() {
+ return jsc;
+ }
+
+ public JavaStreamingContext getStreamingContext() {
+ return jssc;
+ }
+
+ public Pipeline getPipeline() {
+ return pipeline;
+ }
+
+ public PipelineOptions getOptions() {
+ return options;
+ }
+
+ public SerializablePipelineOptions getSerializableOptions() {
+ return serializableOptions;
+ }
+
+ public void setCurrentTransform(AppliedPTransform<?, ?, ?> transform) {
+ this.currentTransform = transform;
+ }
+
+ public AppliedPTransform<?, ?, ?> getCurrentTransform() {
+ return currentTransform;
+ }
+
+ public <T extends PValue> T getInput(PTransform<T, ?> transform) {
+ @SuppressWarnings("unchecked")
+ T input =
+ (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform()));
+ return input;
+ }
+
+ public <T> Map<TupleTag<?>, PValue> getInputs(PTransform<?, ?> transform) {
+ checkArgument(currentTransform != null, "can only be called with non-null currentTransform");
+ checkArgument(
+ currentTransform.getTransform() == transform, "can only be called with current transform");
+ return currentTransform.getInputs();
+ }
+
+ public <T extends PValue> T getOutput(PTransform<?, T> transform) {
+ @SuppressWarnings("unchecked")
+ T output = (T) Iterables.getOnlyElement(getOutputs(transform).values());
+ return output;
+ }
+
+ public Map<TupleTag<?>, PValue> getOutputs(PTransform<?, ?> transform) {
+ checkArgument(currentTransform != null, "can only be called with non-null currentTransform");
+ checkArgument(
+ currentTransform.getTransform() == transform, "can only be called with current transform");
+ return currentTransform.getOutputs();
+ }
+
+ public Map<TupleTag<?>, Coder<?>> getOutputCoders() {
+ return currentTransform
+ .getOutputs()
+ .entrySet()
+ .stream()
+ .filter(e -> e.getValue() instanceof PCollection)
+ .collect(Collectors.toMap(e -> e.getKey(), e -> ((PCollection) e.getValue()).getCoder()));
+ }
+
+ private boolean shouldCache(PValue pvalue) {
+ if ((pvalue instanceof PCollection)
+ && cacheCandidates.containsKey(pvalue)
+ && cacheCandidates.get(pvalue) > 1) {
+ return true;
+ }
+ return false;
+ }
+
+ public void putDataset(
+ PTransform<?, ? extends PValue> transform, Dataset dataset, boolean forceCache) {
+ putDataset(getOutput(transform), dataset, forceCache);
+ }
+
+ public void putDataset(PTransform<?, ? extends PValue> transform, Dataset dataset) {
+ putDataset(transform, dataset, false);
+ }
+
+ public void putDataset(PValue pvalue, Dataset dataset, boolean forceCache) {
+ try {
+ dataset.setName(pvalue.getName());
+ } catch (IllegalStateException e) {
+ // name not set, ignore
+ }
+ if ((forceCache || shouldCache(pvalue)) && pvalue instanceof PCollection) {
+ // we cache only PCollection
+ Coder<?> coder = ((PCollection<?>) pvalue).getCoder();
+ Coder<? extends BoundedWindow> wCoder =
+ ((PCollection<?>) pvalue).getWindowingStrategy().getWindowFn().windowCoder();
+ dataset.cache(storageLevel(), WindowedValue.getFullCoder(coder, wCoder));
+ }
+ datasets.put(pvalue, dataset);
+ leaves.add(dataset);
+ }
+
+ public Dataset borrowDataset(PTransform<? extends PValue, ?> transform) {
+ return borrowDataset(getInput(transform));
+ }
+
+ public Dataset borrowDataset(PValue pvalue) {
+ Dataset dataset = datasets.get(pvalue);
+ leaves.remove(dataset);
+ return dataset;
+ }
+
+ /**
+ * Computes the outputs for all RDDs that are leaves in the DAG and do not have any actions (like
+ * saving to a file) registered on them (i.e. they are performed for side effects).
+ */
+ public void computeOutputs() {
+ for (Dataset dataset : leaves) {
+ dataset.action(); // force computation.
+ }
+ }
+
+ /**
+ * Retrieve an object of Type T associated with the PValue passed in.
+ *
+ * @param value PValue to retrieve associated data for.
+ * @param <T> Type of object to return.
+ * @return Native object.
+ */
+ @SuppressWarnings("TypeParameterUnusedInFormals")
+ public <T> T get(PValue value) {
+ if (pobjects.containsKey(value)) {
+ T result = (T) pobjects.get(value);
+ return result;
+ }
+ if (pcollections.containsKey(value)) {
+ JavaRDD<?> rdd = ((BoundedDataset) pcollections.get(value)).getRDD();
+ T res = (T) Iterables.getOnlyElement(rdd.collect());
+ pobjects.put(value, res);
+ return res;
+ }
+ throw new IllegalStateException("Cannot resolve un-known PObject: " + value);
+ }
+
+ /**
+ * Return the current views creates in the pipeline.
+ *
+ * @return SparkPCollectionView
+ */
+ public SparkPCollectionView getPViews() {
+ return pviews;
+ }
+
+ /**
+ * Adds/Replaces a view to the current views creates in the pipeline.
+ *
+ * @param view - Identifier of the view
+ * @param value - Actual value of the view
+ * @param coder - Coder of the value
+ */
+ public void putPView(
+ PCollectionView<?> view,
+ Iterable<WindowedValue<?>> value,
+ Coder<Iterable<WindowedValue<?>>> coder) {
+ pviews.putPView(view, value, coder);
+ }
+
+ /**
+ * Get the map of cache candidates hold by the evaluation context.
+ *
+ * @return The current {@link Map} of cache candidates.
+ */
+ public Map<PCollection, Long> getCacheCandidates() {
+ return this.cacheCandidates;
+ }
+
+ <T> Iterable<WindowedValue<T>> getWindowedValues(PCollection<T> pcollection) {
+ @SuppressWarnings("unchecked")
+ BoundedDataset<T> boundedDataset = (BoundedDataset<T>) datasets.get(pcollection);
+ leaves.remove(boundedDataset);
+ return boundedDataset.getValues(pcollection);
+ }
+
+ public String storageLevel() {
+ return serializableOptions.get().as(SparkPipelineOptions.class).getStorageLevel();
+ }
+}
[beam] 17/20: Make codestyle and firebug happy
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 74054fecc0a9a98e5147bb5df97212043ecd90ca
Author: Alexey Romanenko <ar...@gmail.com>
AuthorDate: Fri Nov 23 16:10:11 2018 +0100
Make codestyle and firebug happy
---
.../runners/spark/structuredstreaming/SparkPipelineResult.java | 7 +++++++
.../TransformTranslator.java => package-info.java} | 10 ++--------
.../structuredstreaming/translation/PipelineTranslator.java | 2 +-
.../structuredstreaming/translation/TransformTranslator.java | 1 +
.../structuredstreaming/translation/TranslationContext.java | 6 ++++++
.../{TransformTranslator.java => batch/package-info.java} | 10 ++--------
.../{TransformTranslator.java => package-info.java} | 10 ++--------
.../{TransformTranslator.java => streaming/package-info.java} | 10 ++--------
8 files changed, 23 insertions(+), 33 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java
index a8b3640..c55526f 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java
@@ -18,32 +18,39 @@
package org.apache.beam.runners.spark.structuredstreaming;
import java.io.IOException;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.metrics.MetricResults;
import org.joda.time.Duration;
+/** Represents a Spark pipeline execution result. */
public class SparkPipelineResult implements PipelineResult {
+ @Nullable // TODO: remove once method will be implemented
@Override
public State getState() {
return null;
}
+ @Nullable // TODO: remove once method will be implemented
@Override
public State cancel() throws IOException {
return null;
}
+ @Nullable // TODO: remove once method will be implemented
@Override
public State waitUntilFinish(Duration duration) {
return null;
}
+ @Nullable // TODO: remove once method will be implemented
@Override
public State waitUntilFinish() {
return null;
}
+ @Nullable // TODO: remove once method will be implemented
@Override
public MetricResults metrics() {
return null;
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java
similarity index 70%
copy from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
copy to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java
index fc55a9e..aefeb28 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java
@@ -15,12 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.runners.spark.structuredstreaming.translation;
-import org.apache.beam.sdk.transforms.PTransform;
-
-public interface TransformTranslator<TransformT extends PTransform> {
-
- /** Base class for translators of {@link PTransform}. */
- void translateTransform(TransformT transform, TranslationContext context);
-}
+/** Internal implementation of the Beam runner for Apache Spark. */
+package org.apache.beam.runners.spark.structuredstreaming;
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
index bb40631..c771915 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
@@ -133,7 +133,7 @@ public abstract class PipelineTranslator extends Pipeline.PipelineVisitor.Defaul
}
/**
- * get a {@link TransformTranslator} for the given {@link TransformHierarchy.Node}
+ * Get a {@link TransformTranslator} for the given {@link TransformHierarchy.Node}.
*
* @param node
* @return
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
index fc55a9e..f9558c9 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
@@ -19,6 +19,7 @@ package org.apache.beam.runners.spark.structuredstreaming.translation;
import org.apache.beam.sdk.transforms.PTransform;
+/** Supports translation between a Beam transform, and Spark's operations on Datasets. */
public interface TransformTranslator<TransformT extends PTransform> {
/** Base class for translators of {@link PTransform}. */
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
index 8f61d0c..aa831ed 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.runners.spark.structuredstreaming.translation;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import java.util.HashMap;
import java.util.Map;
import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
@@ -33,9 +34,14 @@ import org.apache.spark.sql.SparkSession;
*/
public class TranslationContext {
+ @SuppressFBWarnings("URF_UNREAD_FIELD") // make findbug happy
private AppliedPTransform<?, ?, ?> currentTransform;
+
private final Map<PValue, Dataset<?>> datasets;
+
+ @SuppressFBWarnings("URF_UNREAD_FIELD") // make findbug happy
private SparkSession sparkSession;
+
private final SparkPipelineOptions options;
public void setCurrentTransform(AppliedPTransform<?, ?, ?> currentTransform) {
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/package-info.java
similarity index 76%
copy from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
copy to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/package-info.java
index fc55a9e..6d3ce5a 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/package-info.java
@@ -15,12 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.runners.spark.structuredstreaming.translation;
-import org.apache.beam.sdk.transforms.PTransform;
-
-public interface TransformTranslator<TransformT extends PTransform> {
-
- /** Base class for translators of {@link PTransform}. */
- void translateTransform(TransformT transform, TranslationContext context);
-}
+/** Internal utilities to translate Beam pipelines to Spark batching. */
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/package-info.java
similarity index 77%
copy from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
copy to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/package-info.java
index fc55a9e..2754ac5 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/package-info.java
@@ -15,12 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.runners.spark.structuredstreaming.translation;
-
-import org.apache.beam.sdk.transforms.PTransform;
-public interface TransformTranslator<TransformT extends PTransform> {
-
- /** Base class for translators of {@link PTransform}. */
- void translateTransform(TransformT transform, TranslationContext context);
-}
+/** Internal translators for running Beam pipelines on Spark. */
+package org.apache.beam.runners.spark.structuredstreaming.translation;
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/package-info.java
similarity index 76%
copy from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
copy to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/package-info.java
index fc55a9e..67f3613 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/package-info.java
@@ -15,12 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.runners.spark.structuredstreaming.translation;
-import org.apache.beam.sdk.transforms.PTransform;
-
-public interface TransformTranslator<TransformT extends PTransform> {
-
- /** Base class for translators of {@link PTransform}. */
- void translateTransform(TransformT transform, TranslationContext context);
-}
+/** Internal utilities to translate Beam pipelines to Spark streaming. */
+package org.apache.beam.runners.spark.structuredstreaming.translation.streaming;
[beam] 14/20: Move SparkTransformOverrides to correct package
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit fc03a4af24d05512b5466f98da590118f4ab3ce9
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Thu Nov 22 11:50:45 2018 +0100
Move SparkTransformOverrides to correct package
---
.../spark/structuredstreaming/translation/PipelineTranslator.java | 1 -
.../structuredstreaming/translation/SparkTransformOverrides.java | 4 ++--
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
index 51d65ff..c05fc92 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
@@ -2,7 +2,6 @@ package org.apache.beam.runners.spark.structuredstreaming.translation;
import org.apache.beam.runners.core.construction.PTransformTranslation;
import org.apache.beam.runners.core.construction.PipelineResources;
-import org.apache.beam.runners.spark.SparkTransformOverrides;
import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
import org.apache.beam.runners.spark.structuredstreaming.translation.batch.BatchPipelineTranslator;
import org.apache.beam.runners.spark.structuredstreaming.translation.streaming.StreamingPipelineTranslator;
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java
index 897ac01..8e250bd 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.runners.spark;
+package org.apache.beam.runners.spark.structuredstreaming.translation;
import com.google.common.collect.ImmutableList;
import java.util.List;
@@ -27,7 +27,7 @@ import org.apache.beam.runners.core.construction.UnsupportedOverrideFactory;
import org.apache.beam.sdk.runners.PTransformOverride;
import org.apache.beam.sdk.transforms.PTransform;
-/** {@link PTransform} overrides for Flink runner. */
+/** {@link PTransform} overrides for Spark runner. */
public class SparkTransformOverrides {
public static List<PTransformOverride> getDefaultOverrides(boolean streaming) {
ImmutableList.Builder<PTransformOverride> builder = ImmutableList.builder();
[beam] 10/20: Initialise BatchTranslationContext
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 53b2e71e9e93db052aa3fdc88bd3ad9afcb04274
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Wed Nov 21 12:13:21 2018 +0100
Initialise BatchTranslationContext
---
.../runners/spark/structuredstreaming/SparkRunner.java | 2 +-
.../translation/batch/BatchPipelineTranslator.java | 7 ++++++-
.../translation/batch/BatchTranslationContext.java | 17 ++++++++++++++---
.../streaming/StreamingPipelineTranslator.java | 5 ++++-
4 files changed, 25 insertions(+), 6 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
index ab2215b..de20133 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
@@ -98,7 +98,7 @@ public final class SparkRunner extends PipelineRunner<SparkPipelineResult> {
PipelineTranslator.detectTranslationMode(pipeline, options);
PipelineTranslator.replaceTransforms(pipeline, options);
PipelineTranslator.prepareFilesToStageForRemoteClusterExecution(options);
- PipelineTranslator pipelineTranslator = options.isStreaming() ? new StreamingPipelineTranslator() : new BatchPipelineTranslator();
+ PipelineTranslator pipelineTranslator = options.isStreaming() ? new StreamingPipelineTranslator(options) : new BatchPipelineTranslator(options);
pipelineTranslator.translate(pipeline);
}
private void executePipeline(Pipeline pipeline) {}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
index 2459372..1bf660f 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
@@ -4,10 +4,13 @@ import java.util.HashMap;
import java.util.Map;
import javax.annotation.Nullable;
import org.apache.beam.runners.core.construction.PTransformTranslation;
+import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
import org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,7 +48,9 @@ public class BatchPipelineTranslator extends PipelineTranslator {
}
private static final Logger LOG = LoggerFactory.getLogger(BatchPipelineTranslator.class);
-
+ public BatchPipelineTranslator(SparkPipelineOptions options) {
+ translationContext = new BatchTranslationContext(options);
+ }
/** Returns a translator for the given node, if it is possible, otherwise null. */
private static BatchTransformTranslator<?> getTransformTranslator(TransformHierarchy.Node node) {
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
index 1d991f1..b53aa19 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
@@ -5,6 +5,7 @@ import java.util.Map;
import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.values.PValue;
+import org.apache.spark.SparkConf;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SparkSession;
@@ -20,14 +21,24 @@ public class BatchTranslationContext {
*/
private final Map<PValue, Dataset<?>> danglingDataSets;
- private final SparkSession sparkSession;
+ private SparkSession sparkSession;
private final SparkPipelineOptions options;
private AppliedPTransform<?, ?, ?> currentTransform;
- public BatchTranslationContext(SparkSession sparkSession, SparkPipelineOptions options) {
- this.sparkSession = sparkSession;
+ public BatchTranslationContext(SparkPipelineOptions options) {
+ SparkConf sparkConf = new SparkConf();
+ sparkConf.setMaster(options.getSparkMaster());
+ sparkConf.setAppName(options.getAppName());
+ if (options.getFilesToStage() != null && !options.getFilesToStage().isEmpty()) {
+ sparkConf.setJars(options.getFilesToStage().toArray(new String[0]));
+ }
+
+ SparkSession sparkSession = SparkSession
+ .builder()
+ .config(sparkConf)
+ .getOrCreate();
this.options = options;
this.datasets = new HashMap<>();
this.danglingDataSets = new HashMap<>();
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
index 547083c..7bed930 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
@@ -1,7 +1,10 @@
package org.apache.beam.runners.spark.structuredstreaming.translation.streaming;
+import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
import org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator;
public class StreamingPipelineTranslator extends PipelineTranslator {
-//TODO impl
+
+ public StreamingPipelineTranslator(SparkPipelineOptions options) {
+ }
}
[beam] 05/20: Add global pipeline translation structure
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 999761ac7a3700a31b1dd2dc118c6c944ee93f78
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Wed Nov 21 09:36:32 2018 +0100
Add global pipeline translation structure
---
.../runners/spark/structuredstreaming/SparkRunner.java | 9 ++++-----
.../translation/PipelineTranslator.java | 16 ++++++++++++++--
2 files changed, 18 insertions(+), 7 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
index 62cd7d3..59c08f7 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
@@ -2,15 +2,14 @@ package org.apache.beam.runners.spark.structuredstreaming;
import static org.apache.beam.runners.core.construction.PipelineResources.detectClassPathResourcesToStage;
-import org.apache.beam.runners.spark.structuredstreaming.translation.BatchPipelineTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.batch.BatchPipelineTranslator;
import org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator;
-import org.apache.beam.runners.spark.structuredstreaming.translation.StreamingPipelineTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.streaming.StreamingPipelineTranslator;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineRunner;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.options.PipelineOptionsValidator;
-import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -99,10 +98,10 @@ public final class SparkRunner extends PipelineRunner<SparkPipelineResult> {
PipelineTranslator.detectTranslationMode(pipeline, options);
PipelineTranslator.replaceTransforms(pipeline, options);
PipelineTranslator.prepareFilesToStageForRemoteClusterExecution(options);
- PipelineTranslator translator = options.isStreaming() ? new StreamingPipelineTranslator() : new BatchPipelineTranslator()
+ PipelineTranslator translator = options.isStreaming() ? new StreamingPipelineTranslator() : new BatchPipelineTranslator();
//init translator with subclass based on mode and env
translator.translate(pipeline);
}
- public void executePipeline(Pipeline pipeline) {}
+ private void executePipeline(Pipeline pipeline) {}
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
index f0ce1e5..99621f6 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
@@ -3,6 +3,8 @@ package org.apache.beam.runners.spark.structuredstreaming.translation;
import org.apache.beam.runners.core.construction.PipelineResources;
import org.apache.beam.runners.spark.SparkTransformOverrides;
import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
+import org.apache.beam.runners.spark.structuredstreaming.translation.batch.BatchPipelineTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.streaming.StreamingPipelineTranslator;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.values.PCollection;
@@ -11,7 +13,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Does all the translation work: mode detection, nodes translation.
+ /**
+ * The role of this class is to detect the pipeline mode and to translate the Beam operators to their Spark counterparts. If we have
+ * a streaming job, this is instantiated as a {@link StreamingPipelineTranslator}. In other
+ * case, i.e. for a batch job, a {@link BatchPipelineTranslator} is created. Correspondingly,
*/
public class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
@@ -41,11 +46,18 @@ public class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
TranslationModeDetector detector = new TranslationModeDetector();
pipeline.traverseTopologically(detector);
if (detector.getTranslationMode().equals(TranslationMode.STREAMING)) {
- // set streaming mode if it's a streaming pipeline
options.setStreaming(true);
}
}
+ /**
+ * Translates the pipeline by passing this class as a visitor.
+ *
+ * @param pipeline The pipeline to be translated
+ */
+ public void translate(Pipeline pipeline) {
+ pipeline.traverseTopologically(this);
+ }
[beam] 07/20: Wire node translators with pipeline translator
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit b77c7bbee35b9cf48e2a2994ad7330e7e2fa9ae6
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Wed Nov 21 10:43:19 2018 +0100
Wire node translators with pipeline translator
---
.../translation/PipelineTranslator.java | 15 ++++-
.../translation/batch/BatchPipelineTranslator.java | 66 ++++++++++++++++++++--
.../translation/batch/BatchTranslationContext.java | 3 +
3 files changed, 77 insertions(+), 7 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
index 99621f6..db5c354 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
@@ -21,7 +21,6 @@ import org.slf4j.LoggerFactory;
public class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
-
/**
* Local configurations work in the same JVM and have no problems with improperly formatted files
* on classpath (eg. directories with .class files or empty directories). Prepare files for
@@ -51,6 +50,20 @@ public class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults{
}
/**
+ * Utility formatting method.
+ *
+ * @param n number of spaces to generate
+ * @return String with "|" followed by n spaces
+ */
+ protected static String genSpaces(int n) {
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < n; i++) {
+ builder.append("| ");
+ }
+ return builder.toString();
+ }
+
+ /**
* Translates the pipeline by passing this class as a visitor.
*
* @param pipeline The pipeline to be translated
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
index 2f7ac23..e20e4c0 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
@@ -8,6 +8,8 @@ import org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTra
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.TransformHierarchy;
import org.apache.beam.sdk.transforms.PTransform;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/** {@link Pipeline.PipelineVisitor} for executing a {@link Pipeline} as a Spark batch job. */
@@ -18,6 +20,9 @@ public class BatchPipelineTranslator extends PipelineTranslator {
// Transform Translator Registry
// --------------------------------------------------------------------------------------------
+ private BatchTranslationContext translationContext;
+ private int depth = 0;
+
@SuppressWarnings("rawtypes")
private static final Map<String, BatchTransformTranslator>
TRANSLATORS = new HashMap<>();
@@ -39,6 +44,9 @@ public class BatchPipelineTranslator extends PipelineTranslator {
TRANSLATORS.put(PTransformTranslation.READ_TRANSFORM_URN, new ReadSourceTranslatorBatch());
}
+ private static final Logger LOG = LoggerFactory.getLogger(BatchPipelineTranslator.class);
+
+
/** Returns a translator for the given node, if it is possible, otherwise null. */
private static BatchTransformTranslator<?> getTranslator(TransformHierarchy.Node node) {
@@ -52,15 +60,61 @@ public class BatchPipelineTranslator extends PipelineTranslator {
}
- @Override public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
- return super.enterCompositeTransform(node);
- //TODO impl
+ // --------------------------------------------------------------------------------------------
+ // Pipeline Visitor Methods
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
+ LOG.info("{} enterCompositeTransform- {}", genSpaces(depth), node.getFullName());
+ depth++;
+
+ BatchTransformTranslator<?> translator = getTranslator(node);
+
+ if (translator != null) {
+ translateNode(node, translator);
+ LOG.info("{} translated- {}", genSpaces(depth), node.getFullName());
+ return CompositeBehavior.DO_NOT_ENTER_TRANSFORM;
+ } else {
+ return CompositeBehavior.ENTER_TRANSFORM;
+ }
}
+ @Override
+ public void leaveCompositeTransform(TransformHierarchy.Node node) {
+ depth--;
+ LOG.info("{} leaveCompositeTransform- {}", genSpaces(depth), node.getFullName());
+ }
- @Override public void visitPrimitiveTransform(TransformHierarchy.Node node) {
- super.visitPrimitiveTransform(node);
- //TODO impl
+ @Override
+ public void visitPrimitiveTransform(TransformHierarchy.Node node) {
+ LOG.info("{} visitPrimitiveTransform- {}", genSpaces(depth), node.getFullName());
+
+ // get the transformation corresponding to the node we are
+ // currently visiting and translate it into its Spark alternative.
+ BatchTransformTranslator<?> translator = getTranslator(node);
+ if (translator == null) {
+ String transformUrn = PTransformTranslation.urnForTransform(node.getTransform());
+ throw new UnsupportedOperationException(
+ "The transform " + transformUrn + " is currently not supported.");
+ }
+ translateNode(node, translator);
}
+ private <T extends PTransform<?, ?>> void translateNode(
+ TransformHierarchy.Node node,
+ BatchTransformTranslator<?> translator) {
+
+ @SuppressWarnings("unchecked")
+ T typedTransform = (T) node.getTransform();
+
+ @SuppressWarnings("unchecked")
+ BatchTransformTranslator<T> typedTranslator = (BatchTransformTranslator<T>) translator;
+
+ // create the applied PTransform on the translationContext
+ translationContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
+ typedTranslator.translateNode(typedTransform, translationContext);
}
+
+
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
index 554beea..1d991f1 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
@@ -33,4 +33,7 @@ public class BatchTranslationContext {
this.danglingDataSets = new HashMap<>();
}
+ public void setCurrentTransform(AppliedPTransform<?, ?, ?> currentTransform) {
+ this.currentTransform = currentTransform;
+ }
}
[beam] 15/20: Move common translation context components to
superclass
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 41742be0d1d546db174537df22c365c6d5b9c758
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Thu Nov 22 12:01:28 2018 +0100
Move common translation context components to superclass
---
.../translation/TranslationContext.java | 29 +++++++++++++++++++++-
.../translation/batch/BatchTranslationContext.java | 20 ++-------------
.../streaming/StreamingTranslationContext.java | 6 ++++-
3 files changed, 35 insertions(+), 20 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
index 3dacde4..e651e70 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
@@ -1,17 +1,44 @@
package org.apache.beam.runners.spark.structuredstreaming.translation;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
import org.apache.beam.sdk.runners.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.SparkSession;
/**
- * Base class that gives a context for {@link PTransform} translation.
+ * Base class that gives a context for {@link PTransform} translation: keeping track of the datasets,
+ * the {@link SparkSession}, the current transform being translated.
*/
public class TranslationContext {
private AppliedPTransform<?, ?, ?> currentTransform;
+ private final Map<PValue, Dataset<?>> datasets;
+ private SparkSession sparkSession;
+ private final SparkPipelineOptions options;
public void setCurrentTransform(AppliedPTransform<?, ?, ?> currentTransform) {
this.currentTransform = currentTransform;
}
+ public TranslationContext(SparkPipelineOptions options) {
+ SparkConf sparkConf = new SparkConf();
+ sparkConf.setMaster(options.getSparkMaster());
+ sparkConf.setAppName(options.getAppName());
+ if (options.getFilesToStage() != null && !options.getFilesToStage().isEmpty()) {
+ sparkConf.setJars(options.getFilesToStage().toArray(new String[0]));
+ }
+
+ this.sparkSession = SparkSession
+ .builder()
+ .config(sparkConf)
+ .getOrCreate();
+ this.options = options;
+ this.datasets = new HashMap<>();
+ }
+
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
index f08e33c..02aad71 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
@@ -11,10 +11,9 @@ import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SparkSession;
/**
- * Keeps track of context of the translation.
+ * This class contains only batch specific context components.
*/
public class BatchTranslationContext extends TranslationContext {
- private final Map<PValue, Dataset<?>> datasets;
/**
* For keeping track about which DataSets don't have a successor. We need to terminate these with
@@ -22,23 +21,8 @@ public class BatchTranslationContext extends TranslationContext {
*/
private final Map<PValue, Dataset<?>> danglingDataSets;
- private SparkSession sparkSession;
- private final SparkPipelineOptions options;
-
public BatchTranslationContext(SparkPipelineOptions options) {
- SparkConf sparkConf = new SparkConf();
- sparkConf.setMaster(options.getSparkMaster());
- sparkConf.setAppName(options.getAppName());
- if (options.getFilesToStage() != null && !options.getFilesToStage().isEmpty()) {
- sparkConf.setJars(options.getFilesToStage().toArray(new String[0]));
- }
-
- this.sparkSession = SparkSession
- .builder()
- .config(sparkConf)
- .getOrCreate();
- this.options = options;
- this.datasets = new HashMap<>();
+ super(options);
this.danglingDataSets = new HashMap<>();
}
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java
index f2ee34b..ebccfa7 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingTranslationContext.java
@@ -1,10 +1,14 @@
package org.apache.beam.runners.spark.structuredstreaming.translation.streaming;
+import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
import org.apache.beam.runners.spark.structuredstreaming.translation.TranslationContext;
/**
- * * Keeps track of context of the translation.
+ * This class contains only streaming specific context components.
*/
public class StreamingTranslationContext extends TranslationContext {
+ public StreamingTranslationContext(SparkPipelineOptions options) {
+ super(options);
+ }
}
[beam] 19/20: Post-pone batch qualifier in all classes names for
readability
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit c567d40dde09f92f59601c22c0a4132b62de8da6
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Mon Nov 26 11:14:55 2018 +0100
Post-pone batch qualifier in all classes names for readability
---
.../spark/structuredstreaming/SparkRunner.java | 4 ++--
.../translation/PipelineTranslator.java | 4 ++--
...ator.java => CombinePerKeyTranslatorBatch.java} | 2 +-
...java => FlattenPCollectionTranslatorBatch.java} | 2 +-
...nslator.java => GroupByKeyTranslatorBatch.java} | 2 +-
...DoTranslator.java => ParDoTranslatorBatch.java} | 2 +-
...ranslator.java => PipelineTranslatorBatch.java} | 22 +++++++++++-----------
...nslator.java => ReadSourceTranslatorBatch.java} | 2 +-
...anslator.java => ReshuffleTranslatorBatch.java} | 2 +-
...onContext.java => TranslationContextBatch.java} | 4 ++--
...lator.java => WindowAssignTranslatorBatch.java} | 2 +-
11 files changed, 24 insertions(+), 24 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
index b76a530..e3fd6b4 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkRunner.java
@@ -20,7 +20,7 @@ package org.apache.beam.runners.spark.structuredstreaming;
import static org.apache.beam.runners.core.construction.PipelineResources.detectClassPathResourcesToStage;
import org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator;
-import org.apache.beam.runners.spark.structuredstreaming.translation.batch.BatchPipelineTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.batch.PipelineTranslatorBatch;
import org.apache.beam.runners.spark.structuredstreaming.translation.streaming.StreamingPipelineTranslator;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.PipelineRunner;
@@ -122,7 +122,7 @@ public final class SparkRunner extends PipelineRunner<SparkPipelineResult> {
PipelineTranslator pipelineTranslator =
options.isStreaming()
? new StreamingPipelineTranslator(options)
- : new BatchPipelineTranslator(options);
+ : new PipelineTranslatorBatch(options);
pipelineTranslator.translate(pipeline);
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
index c771915..d64b8b1 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
@@ -20,7 +20,7 @@ package org.apache.beam.runners.spark.structuredstreaming.translation;
import org.apache.beam.runners.core.construction.PTransformTranslation;
import org.apache.beam.runners.core.construction.PipelineResources;
import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
-import org.apache.beam.runners.spark.structuredstreaming.translation.batch.BatchPipelineTranslator;
+import org.apache.beam.runners.spark.structuredstreaming.translation.batch.PipelineTranslatorBatch;
import org.apache.beam.runners.spark.structuredstreaming.translation.streaming.StreamingPipelineTranslator;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.runners.TransformHierarchy;
@@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
* It also does the pipeline preparation: mode detection, transforms replacement, classpath
* preparation. If we have a streaming job, it is instantiated as a {@link
* StreamingPipelineTranslator}. If we have a batch job, it is instantiated as a {@link
- * BatchPipelineTranslator}.
+ * PipelineTranslatorBatch}.
*/
public abstract class PipelineTranslator extends Pipeline.PipelineVisitor.Defaults {
private int depth = 0;
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchCombinePerKeyTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
similarity index 95%
rename from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchCombinePerKeyTranslator.java
rename to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
index 4a10329..c8946d9 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchCombinePerKeyTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
@@ -23,7 +23,7 @@ import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
-class BatchCombinePerKeyTranslator<K, InputT, AccumT, OutputT>
+class CombinePerKeyTranslatorBatch<K, InputT, AccumT, OutputT>
implements TransformTranslator<
PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>>> {
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchFlattenPCollectionTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenPCollectionTranslatorBatch.java
similarity index 97%
rename from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchFlattenPCollectionTranslator.java
rename to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenPCollectionTranslatorBatch.java
index d24f60c..87a250e 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchFlattenPCollectionTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenPCollectionTranslatorBatch.java
@@ -23,7 +23,7 @@ import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionList;
-class BatchFlattenPCollectionTranslator<T>
+class FlattenPCollectionTranslatorBatch<T>
implements TransformTranslator<PTransform<PCollectionList<T>, PCollection<T>>> {
@Override
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchGroupByKeyTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java
similarity index 97%
rename from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchGroupByKeyTranslator.java
rename to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java
index 829ba8a..4ee77fb 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchGroupByKeyTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java
@@ -23,7 +23,7 @@ import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
-class BatchGroupByKeyTranslator<K, InputT>
+class GroupByKeyTranslatorBatch<K, InputT>
implements TransformTranslator<
PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, Iterable<InputT>>>>> {
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchParDoTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java
similarity index 97%
rename from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchParDoTranslator.java
rename to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java
index 56aa504..1e57098 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchParDoTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java
@@ -23,7 +23,7 @@ import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionTuple;
-class BatchParDoTranslator<InputT, OutputT>
+class ParDoTranslatorBatch<InputT, OutputT>
implements TransformTranslator<PTransform<PCollection<InputT>, PCollectionTuple>> {
@Override
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java
similarity index 79%
rename from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
rename to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java
index 6648539..e883131 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java
@@ -30,10 +30,10 @@ import org.apache.beam.sdk.transforms.PTransform;
/**
* {@link PipelineTranslator} for executing a {@link Pipeline} in Spark in batch mode. This contains
- * only the components specific to batch: {@link BatchTranslationContext}, registry of batch {@link
+ * only the components specific to batch: {@link TranslationContextBatch}, registry of batch {@link
* TransformTranslator} and registry lookup code.
*/
-public class BatchPipelineTranslator extends PipelineTranslator {
+public class PipelineTranslatorBatch extends PipelineTranslator {
// --------------------------------------------------------------------------------------------
// Transform Translator Registry
@@ -44,26 +44,26 @@ public class BatchPipelineTranslator extends PipelineTranslator {
static {
TRANSFORM_TRANSLATORS.put(
- PTransformTranslation.COMBINE_PER_KEY_TRANSFORM_URN, new BatchCombinePerKeyTranslator());
+ PTransformTranslation.COMBINE_PER_KEY_TRANSFORM_URN, new CombinePerKeyTranslatorBatch());
TRANSFORM_TRANSLATORS.put(
- PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN, new BatchGroupByKeyTranslator());
- TRANSFORM_TRANSLATORS.put(PTransformTranslation.RESHUFFLE_URN, new BatchReshuffleTranslator());
+ PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN, new GroupByKeyTranslatorBatch());
+ TRANSFORM_TRANSLATORS.put(PTransformTranslation.RESHUFFLE_URN, new ReshuffleTranslatorBatch());
TRANSFORM_TRANSLATORS.put(
- PTransformTranslation.FLATTEN_TRANSFORM_URN, new BatchFlattenPCollectionTranslator());
+ PTransformTranslation.FLATTEN_TRANSFORM_URN, new FlattenPCollectionTranslatorBatch());
TRANSFORM_TRANSLATORS.put(
- PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN, new BatchWindowAssignTranslator());
+ PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN, new WindowAssignTranslatorBatch());
TRANSFORM_TRANSLATORS.put(
- PTransformTranslation.PAR_DO_TRANSFORM_URN, new BatchParDoTranslator());
+ PTransformTranslation.PAR_DO_TRANSFORM_URN, new ParDoTranslatorBatch());
TRANSFORM_TRANSLATORS.put(
- PTransformTranslation.READ_TRANSFORM_URN, new BatchReadSourceTranslator());
+ PTransformTranslation.READ_TRANSFORM_URN, new ReadSourceTranslatorBatch());
}
- public BatchPipelineTranslator(SparkPipelineOptions options) {
- translationContext = new BatchTranslationContext(options);
+ public PipelineTranslatorBatch(SparkPipelineOptions options) {
+ translationContext = new TranslationContextBatch(options);
}
/** Returns a translator for the given node, if it is possible, otherwise null. */
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReadSourceTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java
similarity index 97%
rename from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReadSourceTranslator.java
rename to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java
index d9fcfbb..d18eb2e 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReadSourceTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java
@@ -23,7 +23,7 @@ import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
-class BatchReadSourceTranslator<T>
+class ReadSourceTranslatorBatch<T>
implements TransformTranslator<PTransform<PBegin, PCollection<T>>> {
@Override
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReshuffleTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java
similarity index 95%
rename from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReshuffleTranslator.java
rename to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java
index 1423308..17589ef 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchReshuffleTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java
@@ -21,7 +21,7 @@ import org.apache.beam.runners.spark.structuredstreaming.translation.TransformTr
import org.apache.beam.runners.spark.structuredstreaming.translation.TranslationContext;
import org.apache.beam.sdk.transforms.Reshuffle;
-class BatchReshuffleTranslator<K, InputT> implements TransformTranslator<Reshuffle<K, InputT>> {
+class ReshuffleTranslatorBatch<K, InputT> implements TransformTranslator<Reshuffle<K, InputT>> {
@Override
public void translateTransform(Reshuffle<K, InputT> transform, TranslationContext context) {}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/TranslationContextBatch.java
similarity index 92%
rename from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
rename to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/TranslationContextBatch.java
index 6f50895..e849471 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/TranslationContextBatch.java
@@ -25,7 +25,7 @@ import org.apache.beam.sdk.values.PValue;
import org.apache.spark.sql.Dataset;
/** This class contains only batch specific context components. */
-public class BatchTranslationContext extends TranslationContext {
+public class TranslationContextBatch extends TranslationContext {
/**
* For keeping track about which DataSets don't have a successor. We need to terminate these with
@@ -33,7 +33,7 @@ public class BatchTranslationContext extends TranslationContext {
*/
private final Map<PValue, Dataset<?>> danglingDataSets;
- public BatchTranslationContext(SparkPipelineOptions options) {
+ public TranslationContextBatch(SparkPipelineOptions options) {
super(options);
this.danglingDataSets = new HashMap<>();
}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchWindowAssignTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java
similarity index 97%
rename from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchWindowAssignTranslator.java
rename to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java
index 65a7cae..51e21c2 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchWindowAssignTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java
@@ -22,7 +22,7 @@ import org.apache.beam.runners.spark.structuredstreaming.translation.Translation
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.values.PCollection;
-class BatchWindowAssignTranslator<T>
+class WindowAssignTranslatorBatch<T>
implements TransformTranslator<PTransform<PCollection<T>, PCollection<T>>> {
@Override
[beam] 01/20: Add an empty spark-structured-streaming runner
project targeting spark 2.4.0
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 03d333d62193181e24d8a874068f99796b987d12
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Tue Nov 13 17:07:02 2018 +0100
Add an empty spark-structured-streaming runner project targeting spark 2.4.0
---
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +
runners/spark-structured-streaming/build.gradle | 93 ++++++++++++++++++++++
settings.gradle | 2 +
3 files changed, 97 insertions(+)
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index ba17983..8888cd2 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -321,6 +321,7 @@ class BeamModulePlugin implements Plugin<Project> {
def hadoop_version = "2.7.3"
def jackson_version = "2.9.5"
def spark_version = "2.3.2"
+ def spark_structured_streaming_version = "2.4.0"
def apex_core_version = "3.7.0"
def apex_malhar_version = "3.4.0"
def postgres_version = "42.2.2"
@@ -432,6 +433,7 @@ class BeamModulePlugin implements Plugin<Project> {
slf4j_jdk14 : "org.slf4j:slf4j-jdk14:1.7.25",
slf4j_log4j12 : "org.slf4j:slf4j-log4j12:1.7.25",
snappy_java : "org.xerial.snappy:snappy-java:1.1.4",
+ spark_sql : "org.apache.spark:spark-core_2.11:$spark_structured_streaming_version",
spark_core : "org.apache.spark:spark-core_2.11:$spark_version",
spark_network_common : "org.apache.spark:spark-network-common_2.11:$spark_version",
spark_streaming : "org.apache.spark:spark-streaming_2.11:$spark_version",
diff --git a/runners/spark-structured-streaming/build.gradle b/runners/spark-structured-streaming/build.gradle
new file mode 100644
index 0000000..b33a2b6
--- /dev/null
+++ b/runners/spark-structured-streaming/build.gradle
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import groovy.json.JsonOutput
+
+apply plugin: org.apache.beam.gradle.BeamModulePlugin
+applyJavaNature()
+
+description = "Apache Beam :: Runners :: Spark-Structured-Streaming"
+
+/*
+ * We need to rely on manually specifying these evaluationDependsOn to ensure that
+ * the following projects are evaluated before we evaluate this project. This is because
+ * we are attempting to reference the "sourceSets.test.output" directly.
+ */
+evaluationDependsOn(":beam-sdks-java-core")
+
+configurations {
+ validatesRunner
+}
+
+test {
+ systemProperty "spark.ui.enabled", "false"
+ systemProperty "spark.ui.showConsoleProgress", "false"
+ forkEvery 1
+ maxParallelForks 4
+ useJUnit {
+ //TODO add test excludes
+ }
+}
+
+dependencies {
+ shadow project(path: ":beam-model-pipeline", configuration: "shadow")
+ shadow project(path: ":beam-sdks-java-core", configuration: "shadow")
+ shadow project(path: ":beam-runners-core-construction-java", configuration: "shadow")
+ shadow project(path: ":beam-runners-core-java", configuration: "shadow")
+ shadow library.java.guava
+ shadow library.java.jackson_annotations
+ shadow library.java.slf4j_api
+ shadow library.java.joda_time
+ shadow "io.dropwizard.metrics:metrics-core:3.1.2"
+ shadow library.java.jackson_module_scala
+ provided library.java.spark_sql
+ provided library.java.hadoop_common
+ provided library.java.hadoop_mapreduce_client_core
+ provided library.java.commons_compress
+ provided library.java.commons_lang3
+ provided library.java.commons_io_2x
+ provided library.java.hamcrest_core
+ provided library.java.hamcrest_library
+ provided "org.apache.zookeeper:zookeeper:3.4.11"
+ provided "org.scala-lang:scala-library:2.11.8"
+ provided "com.esotericsoftware.kryo:kryo:2.21"
+ shadowTest project(path: ":beam-sdks-java-io-kafka", configuration: "shadow")
+ shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest")
+ shadowTest project(path: ":beam-runners-core-java", configuration: "shadowTest")
+ shadowTest library.java.avro
+ shadowTest library.java.kafka_clients
+ shadowTest library.java.junit
+ shadowTest library.java.mockito_core
+ shadowTest library.java.jackson_dataformat_yaml
+ shadowTest "org.apache.kafka:kafka_2.11:0.11.0.1"
+ validatesRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest")
+ validatesRunner project(path: project.path, configuration: "shadowTest")
+ validatesRunner project(path: project.path, configuration: "shadow")
+ validatesRunner project(path: project.path, configuration: "provided")
+}
+
+configurations.testRuntimeClasspath {
+ // Testing the Spark runner causes a StackOverflowError if slf4j-jdk14 is on the classpath
+ exclude group: "org.slf4j", module: "slf4j-jdk14"
+}
+
+configurations.validatesRunner {
+ // Testing the Spark runner causes a StackOverflowError if slf4j-jdk14 is on the classpath
+ exclude group: "org.slf4j", module: "slf4j-jdk14"
+}
+
diff --git a/settings.gradle b/settings.gradle
index 7c21cdc..daca389 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -62,6 +62,8 @@ include "beam-runners-reference-job-server"
project(":beam-runners-reference-job-server").dir = file("runners/reference/job-server")
include "beam-runners-spark"
project(":beam-runners-spark").dir = file("runners/spark")
+include "beam-runners-spark-structured-streaming"
+project(":beam-runners-spark-structured-streaming").dir = file("runners/spark-structured-streaming")
include "beam-runners-samza"
project(":beam-runners-samza").dir = file("runners/samza")
include "beam-sdks-go"
[beam] 06/20: Add nodes translators structure
Posted by ec...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 91f283a6fa2c00bf1fdb837441297cb788fd4fbf
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Wed Nov 21 09:36:49 2018 +0100
Add nodes translators structure
---
.../translation/BatchPipelineTranslator.java | 20 -------
.../translation/batch/BatchPipelineTranslator.java | 66 ++++++++++++++++++++++
.../batch/BatchTransformTranslator.java | 11 ++++
.../translation/batch/BatchTranslationContext.java | 36 ++++++++++++
.../batch/CombinePerKeyTranslatorBatch.java | 14 +++++
.../batch/FlattenPCollectionTranslatorBatch.java | 13 +++++
.../batch/GroupByKeyTranslatorBatch.java | 14 +++++
.../translation/batch/ParDoTranslatorBatch.java | 13 +++++
.../batch/ReadSourceTranslatorBatch.java | 12 ++++
.../batch/ReshuffleTranslatorBatch.java | 11 ++++
.../batch/WindowAssignTranslatorBatch.java | 12 ++++
.../StreamingPipelineTranslator.java | 6 +-
12 files changed, 206 insertions(+), 22 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/BatchPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/BatchPipelineTranslator.java
deleted file mode 100644
index e66555c..0000000
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/BatchPipelineTranslator.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package org.apache.beam.runners.spark.structuredstreaming.translation;
-
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.runners.TransformHierarchy;
-import org.apache.beam.sdk.values.PValue;
-
-public class BatchPipelineTranslator extends PipelineTranslator {
-
-
- @Override public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
- return super.enterCompositeTransform(node);
- }
-
-
- @Override public void visitPrimitiveTransform(TransformHierarchy.Node node) {
- super.visitPrimitiveTransform(node);
- }
-
-
-}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
new file mode 100644
index 0000000..2f7ac23
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchPipelineTranslator.java
@@ -0,0 +1,66 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.beam.runners.core.construction.PTransformTranslation;
+import org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.runners.TransformHierarchy;
+import org.apache.beam.sdk.transforms.PTransform;
+
+/** {@link Pipeline.PipelineVisitor} for executing a {@link Pipeline} as a Spark batch job. */
+
+public class BatchPipelineTranslator extends PipelineTranslator {
+
+
+ // --------------------------------------------------------------------------------------------
+ // Transform Translator Registry
+ // --------------------------------------------------------------------------------------------
+
+ @SuppressWarnings("rawtypes")
+ private static final Map<String, BatchTransformTranslator>
+ TRANSLATORS = new HashMap<>();
+
+ static {
+ TRANSLATORS.put(PTransformTranslation.COMBINE_PER_KEY_TRANSFORM_URN,
+ new CombinePerKeyTranslatorBatch());
+ TRANSLATORS
+ .put(PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN, new GroupByKeyTranslatorBatch());
+ TRANSLATORS.put(PTransformTranslation.RESHUFFLE_URN, new ReshuffleTranslatorBatch());
+
+ TRANSLATORS
+ .put(PTransformTranslation.FLATTEN_TRANSFORM_URN, new FlattenPCollectionTranslatorBatch());
+
+ TRANSLATORS
+ .put(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN, new WindowAssignTranslatorBatch());
+
+ TRANSLATORS.put(PTransformTranslation.PAR_DO_TRANSFORM_URN, new ParDoTranslatorBatch());
+
+ TRANSLATORS.put(PTransformTranslation.READ_TRANSFORM_URN, new ReadSourceTranslatorBatch());
+ }
+
+ /** Returns a translator for the given node, if it is possible, otherwise null. */
+ private static BatchTransformTranslator<?> getTranslator(TransformHierarchy.Node node) {
+ @Nullable PTransform<?, ?> transform = node.getTransform();
+ // Root of the graph is null
+ if (transform == null) {
+ return null;
+ }
+ @Nullable String urn = PTransformTranslation.urnForTransformOrNull(transform);
+ return (urn == null) ? null : TRANSLATORS.get(urn);
+ }
+
+
+ @Override public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
+ return super.enterCompositeTransform(node);
+ //TODO impl
+ }
+
+
+ @Override public void visitPrimitiveTransform(TransformHierarchy.Node node) {
+ super.visitPrimitiveTransform(node);
+ //TODO impl
+ }
+
+ }
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTransformTranslator.java
new file mode 100644
index 0000000..ab0cf68
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTransformTranslator.java
@@ -0,0 +1,11 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.sdk.transforms.PTransform;
+
+public interface BatchTransformTranslator<TransformT extends PTransform> {
+
+ /** A translator of a {@link PTransform} in batch mode. */
+
+ void translateNode(TransformT transform, BatchTranslationContext context);
+ }
+
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
new file mode 100644
index 0000000..554beea
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/BatchTranslationContext.java
@@ -0,0 +1,36 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.values.PValue;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.SparkSession;
+
+/**
+ * Keeps track of the {@link Dataset} and the step the translation is in.
+ */
+public class BatchTranslationContext {
+ private final Map<PValue, Dataset<?>> datasets;
+
+ /**
+ * For keeping track about which DataSets don't have a successor. We need to terminate these with
+ * a discarding sink because the Beam model allows dangling operations.
+ */
+ private final Map<PValue, Dataset<?>> danglingDataSets;
+
+ private final SparkSession sparkSession;
+ private final SparkPipelineOptions options;
+
+ private AppliedPTransform<?, ?, ?> currentTransform;
+
+
+ public BatchTranslationContext(SparkSession sparkSession, SparkPipelineOptions options) {
+ this.sparkSession = sparkSession;
+ this.options = options;
+ this.datasets = new HashMap<>();
+ this.danglingDataSets = new HashMap<>();
+ }
+
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
new file mode 100644
index 0000000..6099fbc
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
@@ -0,0 +1,14 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+class CombinePerKeyTranslatorBatch<K, InputT, AccumT, OutputT> implements BatchTransformTranslator<PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>>> {
+
+ @Override public void translateNode(
+ PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>> transform,
+ BatchTranslationContext context) {
+
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenPCollectionTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenPCollectionTranslatorBatch.java
new file mode 100644
index 0000000..281eda9
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenPCollectionTranslatorBatch.java
@@ -0,0 +1,13 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+
+class FlattenPCollectionTranslatorBatch<T> implements BatchTransformTranslator<PTransform<PCollectionList<T>, PCollection<T>>> {
+
+ @Override public void translateNode(PTransform<PCollectionList<T>, PCollection<T>> transform,
+ BatchTranslationContext context) {
+
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java
new file mode 100644
index 0000000..bb0ccc1
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java
@@ -0,0 +1,14 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+class GroupByKeyTranslatorBatch<K, InputT> implements BatchTransformTranslator<PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, Iterable<InputT>>>>> {
+
+ @Override public void translateNode(
+ PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, Iterable<InputT>>>> transform,
+ BatchTranslationContext context) {
+
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java
new file mode 100644
index 0000000..4477853
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java
@@ -0,0 +1,13 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+
+class ParDoTranslatorBatch<InputT, OutputT> implements BatchTransformTranslator<PTransform<PCollection<InputT>, PCollectionTuple>> {
+
+ @Override public void translateNode(PTransform<PCollection<InputT>, PCollectionTuple> transform,
+ BatchTranslationContext context) {
+
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java
new file mode 100644
index 0000000..a30fa70
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java
@@ -0,0 +1,12 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+
+class ReadSourceTranslatorBatch<T> implements BatchTransformTranslator<PTransform<PBegin, PCollection<T>>> {
+
+ @Override public void translateNode(PTransform<PBegin, PCollection<T>> transform, BatchTranslationContext context) {
+
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java
new file mode 100644
index 0000000..6283fdb
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java
@@ -0,0 +1,11 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.sdk.transforms.Reshuffle;
+
+class ReshuffleTranslatorBatch<K, InputT> implements BatchTransformTranslator<Reshuffle<K, InputT>> {
+
+ @Override public void translateNode(Reshuffle<K, InputT> transform,
+ BatchTranslationContext context) {
+
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java
new file mode 100644
index 0000000..21b71b9
--- /dev/null
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java
@@ -0,0 +1,12 @@
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
+
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.values.PCollection;
+
+class WindowAssignTranslatorBatch<T> implements BatchTransformTranslator<PTransform<PCollection<T>, PCollection<T>>> {
+
+ @Override public void translateNode(PTransform<PCollection<T>, PCollection<T>> transform,
+ BatchTranslationContext context) {
+
+ }
+}
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/StreamingPipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
similarity index 53%
rename from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/StreamingPipelineTranslator.java
rename to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
index 2058b37..547083c 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/StreamingPipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/StreamingPipelineTranslator.java
@@ -1,5 +1,7 @@
-package org.apache.beam.runners.spark.structuredstreaming.translation;
+package org.apache.beam.runners.spark.structuredstreaming.translation.streaming;
-public class StreamingPipelineTranslator extends PipelineTranslator {
+import org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator;
+public class StreamingPipelineTranslator extends PipelineTranslator {
+//TODO impl
}