You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2022/07/23 04:18:17 UTC
[beam] branch nightly-refs/heads/master updated (f2f239a44f4 -> 54b0784da7c)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from f2f239a44f4 Merge pull request #17775 from nbali/BEAM-14525
add 0581c49575e Bump terser from 5.9.0 to 5.14.2 in /sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel (#22365)
add 72127f93f45 Fixes #22156: Fix Spark3 runner to compile against Spark 3.2/3.3 and add version tests to verify compatibility going forward (#22157)
add 8d59cf7de71 Closes #22407: Separate sources for SparkStructuredStreamingRunner for Spark 2 & 3 to allow for easier improvements for Spark 3.
add 79d677fed70 Add deprecation warning for Spark 2 in SparkStructuredStreamingRunner
add b9f6af54d52 Merge pull request #22408 from mosche/22407-separate-spark-ssrunner-sources
add 23e10ebc5d7 Bump cloud.google.com/go/storage from 1.23.0 to 1.24.0 in /sdks (#22377)
add 001dd3782bb [Website] change getting window width method
add 1cd5d7339ff Merge pull request #22256: [Website] change getting window width method
add 94941667e6b Drop timeseries:postCommit dependency (#22414)
add 68ff1c79d7a Removing experimental annotation from JdbcIO
add 52a8af59e3d Merge pull request #22412 from Removing experimental annotation from JdbcIO
add df99d6a2eb9 Deduplicate identical environments in a pipeline. (#22308)
add 79db4d25cc8 Skip failing torch post commit test (#22418)
add a6f07296fee Log level fix on local runner (#22420)
add d050a088a3d Update element_type inference (default_type_hints) for batched DoFns with yields_batches/yields_elements (#22198)
add 54b0784da7c Remove spaces in experiments (#22423)
No new revisions were added by this update.
Summary of changes:
.../jenkins/job_LoadTests_Combine_Python.groovy | 2 +-
.../jenkins/job_LoadTests_GBK_Python.groovy | 2 +-
.../job_LoadTests_GBK_Python_reiterate.groovy | 2 +-
.../jenkins/job_LoadTests_ParDo_Python.groovy | 2 +-
.../jenkins/job_LoadTests_SideInput_Python.groovy | 2 +-
.../job_PreCommit_Java_Spark3_Versions.groovy | 25 +-
build.gradle.kts | 2 +-
.../spark/structuredstreaming/Constants.java | 0
.../SparkStructuredStreamingPipelineOptions.java | 7 +
.../SparkStructuredStreamingPipelineResult.java | 39 +--
.../SparkStructuredStreamingRunner.java | 13 +-
.../SparkStructuredStreamingRunnerRegistrar.java | 0
.../aggregators/AggregatorsAccumulator.java | 0
.../aggregators/NamedAggregators.java | 0
.../aggregators/NamedAggregatorsAccumulator.java | 0
.../aggregators/package-info.java | 0
.../structuredstreaming/examples/WordCount.java | 0
.../metrics/AggregatorMetric.java | 74 ++++++
.../metrics/AggregatorMetricSource.java | 0
.../structuredstreaming/metrics/BeamMetricSet.java | 60 +++++
.../metrics/CompositeSource.java | 0
.../metrics/MetricsAccumulator.java | 0
.../MetricsContainerStepMapAccumulator.java | 0
.../metrics/SparkBeamMetric.java | 107 ++++++++
.../metrics/SparkBeamMetricSource.java | 0
.../metrics/SparkMetricsContainerStepMap.java | 0
.../metrics/WithMetricsSupport.java | 91 +++++++
.../structuredstreaming/metrics/package-info.java | 0
.../metrics/sink/CodahaleCsvSink.java | 86 +++++++
.../metrics/sink/CodahaleGraphiteSink.java | 89 +++++++
.../metrics/sink/package-info.java | 0
.../spark/structuredstreaming/package-info.java | 0
.../translation/AbstractTranslationContext.java | 25 +-
.../translation/PipelineTranslator.java | 0
.../translation/SparkSessionFactory.java | 71 ++++++
.../translation/SparkTransformOverrides.java | 0
.../translation/TransformTranslator.java | 0
.../translation/batch/AggregatorCombiner.java | 0
.../batch/CombinePerKeyTranslatorBatch.java | 0
.../CreatePCollectionViewTranslatorBatch.java | 0
.../translation/batch/DoFnFunction.java | 0
.../translation/batch/DoFnRunnerWithMetrics.java | 0
.../translation/batch/FlattenTranslatorBatch.java | 0
.../batch/GroupByKeyTranslatorBatch.java | 0
.../translation/batch/ImpulseTranslatorBatch.java | 0
.../translation/batch/ParDoTranslatorBatch.java | 0
.../translation/batch/PipelineTranslatorBatch.java | 0
.../translation/batch/ProcessContext.java | 0
.../batch/ReadSourceTranslatorBatch.java | 0
.../batch/ReshuffleTranslatorBatch.java | 0
.../batch/WindowAssignTranslatorBatch.java | 0
.../GroupAlsoByWindowViaOutputBufferFn.java | 0
.../batch/functions/NoOpStepContext.java | 0
.../batch/functions/SparkSideInputReader.java | 0
.../translation/batch/functions/package-info.java | 0
.../translation/batch/package-info.java | 0
.../translation/helpers/CoderHelpers.java | 0
.../translation/helpers/EncoderFactory.java | 43 ++--
.../translation/helpers/EncoderHelpers.java | 71 ++++++
.../translation/helpers/KVHelpers.java | 0
.../translation/helpers/MultiOutputCoder.java | 0
.../translation/helpers/RowHelpers.java | 0
.../translation/helpers/SchemaHelpers.java | 0
.../translation/helpers/SideInputBroadcast.java | 0
.../translation/helpers/WindowingHelpers.java | 0
.../translation/helpers/package-info.java | 0
.../translation/package-info.java | 0
.../streaming/PipelineTranslatorStreaming.java | 0
.../streaming/ReadSourceTranslatorStreaming.java | 0
.../translation/streaming/package-info.java | 0
.../translation/utils/CachedSideInputReader.java | 0
.../translation/utils/SideInputStorage.java | 0
.../translation/utils/package-info.java | 0
.../structuredstreaming/SparkSessionRule.java | 21 +-
...parkStructuredStreamingRunnerRegistrarTest.java | 0
.../StructuredStreamingPipelineStateTest.java | 0
.../aggregators/metrics/sink/InMemoryMetrics.java | 36 ++-
.../metrics/sink/InMemoryMetricsSinkRule.java | 0
.../metrics/sink/SparkMetricsSinkTest.java | 45 ++--
.../metrics/SparkBeamMetricTest.java} | 22 +-
.../translation/batch/CombineTest.java | 0
.../translation/batch/ComplexSourceTest.java | 0
.../translation/batch/FlattenTest.java | 0
.../translation/batch/GroupByKeyTest.java | 0
.../translation/batch/ParDoTest.java | 0
.../translation/batch/SimpleSourceTest.java | 0
.../translation/batch/WindowAssignTest.java | 0
.../translation/helpers/EncoderHelpersTest.java | 98 ++++++++
.../translation/streaming/SimpleSourceTest.java | 0
.../utils/SerializationDebugger.java | 0
.../structuredstreaming/utils/package-info.java | 0
runners/spark/3/build.gradle | 34 +++
.../spark/structuredstreaming/Constants.java | 0
.../SparkStructuredStreamingPipelineOptions.java | 7 +
.../SparkStructuredStreamingPipelineResult.java | 39 +--
.../SparkStructuredStreamingRunner.java | 9 +-
.../SparkStructuredStreamingRunnerRegistrar.java | 0
.../aggregators/AggregatorsAccumulator.java | 0
.../aggregators/NamedAggregators.java | 0
.../aggregators/NamedAggregatorsAccumulator.java | 0
.../aggregators/package-info.java | 0
.../structuredstreaming/examples/WordCount.java | 0
.../metrics/AggregatorMetric.java | 74 ++++++
.../metrics/AggregatorMetricSource.java | 0
.../structuredstreaming/metrics/BeamMetricSet.java | 60 +++++
.../metrics/CompositeSource.java | 0
.../metrics/MetricsAccumulator.java | 0
.../MetricsContainerStepMapAccumulator.java | 0
.../metrics/SparkBeamMetric.java | 107 ++++++++
.../metrics/SparkBeamMetricSource.java | 0
.../metrics/SparkMetricsContainerStepMap.java | 0
.../metrics/WithMetricsSupport.java | 91 +++++++
.../structuredstreaming/metrics/package-info.java | 0
.../metrics/sink/CodahaleCsvSink.java | 86 +++++++
.../metrics/sink/CodahaleGraphiteSink.java | 89 +++++++
.../metrics/sink/package-info.java | 0
.../spark/structuredstreaming/package-info.java | 0
.../translation/AbstractTranslationContext.java | 25 +-
.../translation/PipelineTranslator.java | 0
.../translation/SparkSessionFactory.java | 71 ++++++
.../translation/SparkTransformOverrides.java | 0
.../translation/TransformTranslator.java | 0
.../translation/batch/AggregatorCombiner.java | 0
.../batch/CombinePerKeyTranslatorBatch.java | 0
.../CreatePCollectionViewTranslatorBatch.java | 0
.../translation/batch/DatasetSourceBatch.java | 2 +-
.../translation/batch/DoFnFunction.java | 0
.../translation/batch/DoFnRunnerWithMetrics.java | 0
.../translation/batch/FlattenTranslatorBatch.java | 0
.../batch/GroupByKeyTranslatorBatch.java | 0
.../translation/batch/ImpulseTranslatorBatch.java | 0
.../translation/batch/ParDoTranslatorBatch.java | 0
.../translation/batch/PipelineTranslatorBatch.java | 0
.../translation/batch/ProcessContext.java | 0
.../batch/ReadSourceTranslatorBatch.java | 0
.../batch/ReshuffleTranslatorBatch.java | 0
.../batch/WindowAssignTranslatorBatch.java | 0
.../GroupAlsoByWindowViaOutputBufferFn.java | 0
.../batch/functions/NoOpStepContext.java | 0
.../batch/functions/SparkSideInputReader.java | 0
.../translation/batch/functions/package-info.java | 0
.../translation/batch/package-info.java | 0
.../translation/helpers/CoderHelpers.java | 0
.../translation/helpers/EncoderFactory.java | 59 +++--
.../translation/helpers/EncoderHelpers.java | 71 ++++++
.../translation/helpers/KVHelpers.java | 0
.../translation/helpers/MultiOutputCoder.java | 0
.../translation/helpers/RowHelpers.java | 0
.../translation/helpers/SchemaHelpers.java | 0
.../translation/helpers/SideInputBroadcast.java | 0
.../translation/helpers/WindowingHelpers.java | 0
.../translation/helpers/package-info.java | 0
.../translation/package-info.java | 0
.../streaming/PipelineTranslatorStreaming.java | 0
.../streaming/ReadSourceTranslatorStreaming.java | 0
.../translation/streaming/package-info.java | 0
.../translation/utils/CachedSideInputReader.java | 0
.../translation/utils/SideInputStorage.java | 0
.../translation/utils/package-info.java | 0
.../structuredstreaming/SparkSessionRule.java | 21 +-
...parkStructuredStreamingRunnerRegistrarTest.java | 0
.../StructuredStreamingPipelineStateTest.java | 0
.../aggregators/metrics/sink/InMemoryMetrics.java | 36 ++-
.../metrics/sink/InMemoryMetricsSinkRule.java | 0
.../metrics/sink/SparkMetricsSinkTest.java | 45 ++--
.../metrics/SparkBeamMetricTest.java} | 22 +-
.../translation/batch/CombineTest.java | 0
.../translation/batch/ComplexSourceTest.java | 0
.../translation/batch/FlattenTest.java | 0
.../translation/batch/GroupByKeyTest.java | 0
.../translation/batch/ParDoTest.java | 0
.../translation/batch/SimpleSourceTest.java | 0
.../translation/batch/WindowAssignTest.java | 0
.../translation/helpers/EncoderHelpersTest.java | 98 ++++++++
.../translation/streaming/SimpleSourceTest.java | 0
.../utils/SerializationDebugger.java | 0
.../structuredstreaming/utils/package-info.java | 0
runners/spark/spark_runner.gradle | 48 ++--
.../runners/spark/metrics/AggregatorMetric.java | 47 +++-
.../beam/runners/spark/metrics/BeamMetricSet.java | 59 +++++
.../runners/spark/metrics/SparkBeamMetric.java | 93 ++++---
.../runners/spark/metrics/WithMetricsSupport.java | 123 ++-------
.../beam/runners/spark/metrics/sink/CsvSink.java | 59 ++++-
.../runners/spark/metrics/sink/GraphiteSink.java | 66 ++++-
.../metrics/AggregatorMetric.java | 39 ---
.../metrics/SparkBeamMetric.java | 89 -------
.../metrics/WithMetricsSupport.java | 182 --------------
.../metrics/sink/CodahaleCsvSink.java | 36 ---
.../metrics/sink/CodahaleGraphiteSink.java | 34 ---
.../translation/helpers/EncoderHelpers.java | 274 ---------------------
.../aggregators/metrics/sink/InMemoryMetrics.java | 33 +--
.../metrics/sink/SparkMetricsSinkTest.java | 6 +-
.../runners/spark/metrics/SparkBeamMetricTest.java | 18 +-
.../translation/helpers/EncoderHelpersTest.java | 47 ----
.../spark/src/test/resources/metrics.properties | 68 -----
sdks/go.mod | 3 +-
sdks/go.sum | 4 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 5 -
.../ml/inference/pytorch_inference_it_test.py | 1 +
sdks/python/apache_beam/pipeline.py | 44 +++-
sdks/python/apache_beam/pipeline_test.py | 81 +++++-
.../apache-beam-jupyterlab-sidepanel/yarn.lock | 89 ++++---
.../portability/fn_api_runner/worker_handlers.py | 15 +-
.../runners/portability/local_job_service.py | 8 +-
.../apache_beam/runners/worker/log_handler.py | 38 ++-
.../apache_beam/transforms/batch_dofn_test.py | 73 +++++-
sdks/python/apache_beam/transforms/core.py | 44 +++-
sdks/python/apache_beam/typehints/decorators.py | 10 +
.../apache_beam/typehints/decorators_test.py | 40 +++
sdks/python/apache_beam/utils/subprocess_server.py | 5 +-
website/www/site/assets/js/page-nav.js | 2 +-
.../site/content/en/documentation/runners/spark.md | 6 +-
212 files changed, 2527 insertions(+), 1343 deletions(-)
copy runners/spark/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetricsSinkRule.java => .test-infra/jenkins/job_PreCommit_Java_Spark3_Versions.groovy (63%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/Constants.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineOptions.java (85%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineResult.java (83%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunner.java (95%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrar.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/AggregatorsAccumulator.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/NamedAggregators.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/NamedAggregatorsAccumulator.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/package-info.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/examples/WordCount.java (100%)
create mode 100644 runners/spark/2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/AggregatorMetric.java
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/AggregatorMetricSource.java (100%)
create mode 100644 runners/spark/2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/BeamMetricSet.java
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/CompositeSource.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsAccumulator.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsContainerStepMapAccumulator.java (100%)
create mode 100644 runners/spark/2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetric.java
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetricSource.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkMetricsContainerStepMap.java (100%)
create mode 100644 runners/spark/2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/WithMetricsSupport.java
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/package-info.java (100%)
create mode 100644 runners/spark/2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/CodahaleCsvSink.java
create mode 100644 runners/spark/2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/CodahaleGraphiteSink.java
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/package-info.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/AbstractTranslationContext.java (88%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java (100%)
create mode 100644 runners/spark/2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkSessionFactory.java
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/AggregatorCombiner.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CreatePCollectionViewTranslatorBatch.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnFunction.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnRunnerWithMetrics.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTranslatorBatch.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ImpulseTranslatorBatch.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ProcessContext.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/GroupAlsoByWindowViaOutputBufferFn.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/NoOpStepContext.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SparkSideInputReader.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/package-info.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/package-info.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/CoderHelpers.java (100%)
create mode 100644 runners/spark/2/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpers.java
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/KVHelpers.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/MultiOutputCoder.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/RowHelpers.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/SchemaHelpers.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/SideInputBroadcast.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/WindowingHelpers.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/package-info.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/package-info.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/PipelineTranslatorStreaming.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/ReadSourceTranslatorStreaming.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/package-info.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/CachedSideInputReader.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/SideInputStorage.java (100%)
copy runners/spark/{ => 2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/package-info.java (100%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/SparkSessionRule.java (75%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrarTest.java (100%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/StructuredStreamingPipelineStateTest.java (100%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetrics.java (75%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetricsSinkRule.java (100%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/SparkMetricsSinkTest.java (65%)
copy runners/spark/{src/test/java/org/apache/beam/runners/spark/structuredstreaming/metrics/BeamMetricTest.java => 2/src/test/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetricTest.java} (71%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineTest.java (100%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ComplexSourceTest.java (100%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTest.java (100%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTest.java (100%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTest.java (100%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/SimpleSourceTest.java (100%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTest.java (100%)
create mode 100644 runners/spark/2/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpersTest.java
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/SimpleSourceTest.java (100%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/SerializationDebugger.java (100%)
copy runners/spark/{ => 2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/package-info.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/Constants.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineOptions.java (85%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineResult.java (83%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunner.java (97%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrar.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/AggregatorsAccumulator.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/NamedAggregators.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/NamedAggregatorsAccumulator.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/package-info.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/examples/WordCount.java (100%)
create mode 100644 runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/AggregatorMetric.java
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/AggregatorMetricSource.java (100%)
create mode 100644 runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/BeamMetricSet.java
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/CompositeSource.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsAccumulator.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsContainerStepMapAccumulator.java (100%)
create mode 100644 runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetric.java
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetricSource.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkMetricsContainerStepMap.java (100%)
create mode 100644 runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/WithMetricsSupport.java
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/package-info.java (100%)
create mode 100644 runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/CodahaleCsvSink.java
create mode 100644 runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/CodahaleGraphiteSink.java
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/package-info.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/AbstractTranslationContext.java (88%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java (100%)
create mode 100644 runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkSessionFactory.java
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/AggregatorCombiner.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CreatePCollectionViewTranslatorBatch.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnFunction.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnRunnerWithMetrics.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTranslatorBatch.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ImpulseTranslatorBatch.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ProcessContext.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/GroupAlsoByWindowViaOutputBufferFn.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/NoOpStepContext.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SparkSideInputReader.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/package-info.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/package-info.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/CoderHelpers.java (100%)
create mode 100644 runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpers.java
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/KVHelpers.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/MultiOutputCoder.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/RowHelpers.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/SchemaHelpers.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/SideInputBroadcast.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/WindowingHelpers.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/package-info.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/package-info.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/PipelineTranslatorStreaming.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/ReadSourceTranslatorStreaming.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/package-info.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/CachedSideInputReader.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/SideInputStorage.java (100%)
rename runners/spark/{ => 3}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/package-info.java (100%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/SparkSessionRule.java (75%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrarTest.java (100%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/StructuredStreamingPipelineStateTest.java (100%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetrics.java (75%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetricsSinkRule.java (100%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/SparkMetricsSinkTest.java (65%)
rename runners/spark/{src/test/java/org/apache/beam/runners/spark/structuredstreaming/metrics/BeamMetricTest.java => 3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetricTest.java} (71%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineTest.java (100%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ComplexSourceTest.java (100%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTest.java (100%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTest.java (100%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTest.java (100%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/SimpleSourceTest.java (100%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTest.java (100%)
create mode 100644 runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpersTest.java
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/SimpleSourceTest.java (100%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/SerializationDebugger.java (100%)
rename runners/spark/{ => 3}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/package-info.java (100%)
create mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/metrics/BeamMetricSet.java
delete mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/AggregatorMetric.java
delete mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetric.java
delete mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/WithMetricsSupport.java
delete mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/CodahaleCsvSink.java
delete mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/CodahaleGraphiteSink.java
delete mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpers.java
delete mode 100644 runners/spark/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpersTest.java
delete mode 100644 runners/spark/src/test/resources/metrics.properties