You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ec...@apache.org on 2019/01/04 10:38:39 UTC
[beam] 17/50: Make codestyle and firebug happy
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 91f9ef55503cc565c82605e7459e9baa4e6d8ff8
Author: Alexey Romanenko <ar...@gmail.com>
AuthorDate: Fri Nov 23 16:10:11 2018 +0100
Make codestyle and firebug happy
---
.../runners/spark/structuredstreaming/SparkPipelineResult.java | 7 +++++++
.../TransformTranslator.java => package-info.java} | 10 ++--------
.../structuredstreaming/translation/PipelineTranslator.java | 2 +-
.../structuredstreaming/translation/TransformTranslator.java | 1 +
.../structuredstreaming/translation/TranslationContext.java | 6 ++++++
.../{TransformTranslator.java => batch/package-info.java} | 10 ++--------
.../{TransformTranslator.java => package-info.java} | 10 ++--------
.../{TransformTranslator.java => streaming/package-info.java} | 10 ++--------
8 files changed, 23 insertions(+), 33 deletions(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java
index a8b3640..c55526f 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkPipelineResult.java
@@ -18,32 +18,39 @@
package org.apache.beam.runners.spark.structuredstreaming;
import java.io.IOException;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.metrics.MetricResults;
import org.joda.time.Duration;
+/** Represents a Spark pipeline execution result. */
public class SparkPipelineResult implements PipelineResult {
+ @Nullable // TODO: remove once method will be implemented
@Override
public State getState() {
return null;
}
+ @Nullable // TODO: remove once method will be implemented
@Override
public State cancel() throws IOException {
return null;
}
+ @Nullable // TODO: remove once method will be implemented
@Override
public State waitUntilFinish(Duration duration) {
return null;
}
+ @Nullable // TODO: remove once method will be implemented
@Override
public State waitUntilFinish() {
return null;
}
+ @Nullable // TODO: remove once method will be implemented
@Override
public MetricResults metrics() {
return null;
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java
similarity index 70%
copy from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
copy to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java
index fc55a9e..aefeb28 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java
@@ -15,12 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.runners.spark.structuredstreaming.translation;
-import org.apache.beam.sdk.transforms.PTransform;
-
-public interface TransformTranslator<TransformT extends PTransform> {
-
- /** Base class for translators of {@link PTransform}. */
- void translateTransform(TransformT transform, TranslationContext context);
-}
+/** Internal implementation of the Beam runner for Apache Spark. */
+package org.apache.beam.runners.spark.structuredstreaming;
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
index bb40631..c771915 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
@@ -133,7 +133,7 @@ public abstract class PipelineTranslator extends Pipeline.PipelineVisitor.Defaul
}
/**
- * get a {@link TransformTranslator} for the given {@link TransformHierarchy.Node}
+ * Get a {@link TransformTranslator} for the given {@link TransformHierarchy.Node}.
*
* @param node
* @return
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
index fc55a9e..f9558c9 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
@@ -19,6 +19,7 @@ package org.apache.beam.runners.spark.structuredstreaming.translation;
import org.apache.beam.sdk.transforms.PTransform;
+/** Supports translation between a Beam transform, and Spark's operations on Datasets. */
public interface TransformTranslator<TransformT extends PTransform> {
/** Base class for translators of {@link PTransform}. */
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
index 8f61d0c..aa831ed 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.runners.spark.structuredstreaming.translation;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import java.util.HashMap;
import java.util.Map;
import org.apache.beam.runners.spark.structuredstreaming.SparkPipelineOptions;
@@ -33,9 +34,14 @@ import org.apache.spark.sql.SparkSession;
*/
public class TranslationContext {
+ @SuppressFBWarnings("URF_UNREAD_FIELD") // make findbug happy
private AppliedPTransform<?, ?, ?> currentTransform;
+
private final Map<PValue, Dataset<?>> datasets;
+
+ @SuppressFBWarnings("URF_UNREAD_FIELD") // make findbug happy
private SparkSession sparkSession;
+
private final SparkPipelineOptions options;
public void setCurrentTransform(AppliedPTransform<?, ?, ?> currentTransform) {
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/package-info.java
similarity index 76%
copy from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
copy to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/package-info.java
index fc55a9e..6d3ce5a 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/package-info.java
@@ -15,12 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.runners.spark.structuredstreaming.translation;
-import org.apache.beam.sdk.transforms.PTransform;
-
-public interface TransformTranslator<TransformT extends PTransform> {
-
- /** Base class for translators of {@link PTransform}. */
- void translateTransform(TransformT transform, TranslationContext context);
-}
+/** Internal utilities to translate Beam pipelines to Spark batching. */
+package org.apache.beam.runners.spark.structuredstreaming.translation.batch;
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/package-info.java
similarity index 77%
copy from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
copy to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/package-info.java
index fc55a9e..2754ac5 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/package-info.java
@@ -15,12 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.runners.spark.structuredstreaming.translation;
-
-import org.apache.beam.sdk.transforms.PTransform;
-public interface TransformTranslator<TransformT extends PTransform> {
-
- /** Base class for translators of {@link PTransform}. */
- void translateTransform(TransformT transform, TranslationContext context);
-}
+/** Internal translators for running Beam pipelines on Spark. */
+package org.apache.beam.runners.spark.structuredstreaming.translation;
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/package-info.java
similarity index 76%
copy from runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
copy to runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/package-info.java
index fc55a9e..67f3613 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/package-info.java
@@ -15,12 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.runners.spark.structuredstreaming.translation;
-import org.apache.beam.sdk.transforms.PTransform;
-
-public interface TransformTranslator<TransformT extends PTransform> {
-
- /** Base class for translators of {@link PTransform}. */
- void translateTransform(TransformT transform, TranslationContext context);
-}
+/** Internal utilities to translate Beam pipelines to Spark streaming. */
+package org.apache.beam.runners.spark.structuredstreaming.translation.streaming;