You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by bh...@apache.org on 2020/11/25 00:33:24 UTC
[beam] 01/01: Merge remote-tracking branch 'origin/master' into
website-revamp
This is an automated email from the ASF dual-hosted git repository.
bhulette pushed a commit to branch website-revamp
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 0850259add2661e5d169dd939b18c270b5ffefcf
Merge: 1f790fc d7655c1
Author: Brian Hulette <bh...@google.com>
AuthorDate: Tue Nov 24 16:28:51 2020 -0800
Merge remote-tracking branch 'origin/master' into website-revamp
.github/PULL_REQUEST_TEMPLATE.md | 4 +-
.github/workflows/build_wheels.yml | 8 +-
.github/workflows/java_tests.yml | 2 +-
.github/workflows/python_tests.yml | 2 +-
.test-infra/jenkins/CommonTestProperties.groovy | 12 +-
...Properties.groovy => JavaTestProperties.groovy} | 4 +-
.test-infra/jenkins/LoadTestsBuilder.groovy | 15 +-
.test-infra/jenkins/PythonTestProperties.groovy | 10 +
.test-infra/jenkins/README.md | 6 +-
.../jenkins/job_LoadTests_ParDo_Flink_Go.groovy | 150 +++++
...it_CrossLanguageValidatesRunner_Dataflow.groovy | 38 +-
...mmit_CrossLanguageValidatesRunner_Direct.groovy | 23 +-
...ommit_CrossLanguageValidatesRunner_Flink.groovy | 23 +-
...ommit_CrossLanguageValidatesRunner_Spark.groovy | 23 +-
...tCommit_Java_Examples_Dataflow_V2_Java11.groovy | 50 ++
..._PreCommit_Java_Examples_Dataflow_Java11.groovy | 1 -
.../jenkins/job_Publish_Docker_Snapshots.groovy | 8 +-
.../jenkins/job_Publish_SDK_Image_Snapshots.groovy | 55 ++
CHANGES.md | 9 +-
README.md | 4 +-
build.gradle | 28 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 20 +-
.../apache/beam/gradle/GrpcVendoring_1_26_0.groovy | 2 +-
learning/katas/go/README.md | 29 +
.../src/main/scripts/build_release_candidate.sh | 6 +-
release/src/main/scripts/publish_docker_images.sh | 2 +-
.../core/construction/PTransformMatchers.java | 2 +-
runners/flink/job-server/flink_job_server.gradle | 5 +-
runners/google-cloud-dataflow-java/build.gradle | 78 ++-
.../dataflow/DataflowPipelineTranslator.java | 14 +-
.../beam/runners/dataflow/DataflowRunner.java | 29 +-
.../runners/dataflow/GroupIntoBatchesOverride.java | 132 ++++-
.../beam/runners/dataflow/util/PropertyNames.java | 1 +
.../dataflow/DataflowPipelineTranslatorTest.java | 67 ++-
.../beam/runners/dataflow/DataflowRunnerTest.java | 64 ++-
.../dataflow/worker/WindmillTimerInternals.java | 13 +-
.../dataflow/worker/WorkItemStatusClient.java | 2 +-
.../worker/WindmillTimerInternalsTest.java | 4 +-
runners/portability/java/build.gradle | 9 +-
runners/spark/build.gradle | 66 ++-
sdks/go/pkg/beam/core/metrics/metrics.go | 118 ++++
.../core/runtime/graphx/schema/logicaltypes.go | 121 ++++
.../pkg/beam/core/runtime/graphx/schema/schema.go | 115 ++--
.../beam/core/runtime/graphx/schema/schema_test.go | 55 +-
sdks/go/pkg/beam/core/runtime/graphx/translate.go | 44 +-
sdks/go/pkg/beam/core/runtime/graphx/xlang.go | 210 +++++++
sdks/go/pkg/beam/core/runtime/graphx/xlang_test.go | 218 +++++++
.../go/pkg/beam/core/runtime/harness/monitoring.go | 187 +-----
.../beam/core/runtime/harness/monitoring_test.go | 33 +-
sdks/go/pkg/beam/core/runtime/metricsx/metricsx.go | 172 ++++++
.../beam/core/runtime/metricsx/metricsx_test.go | 166 ++++++
sdks/go/pkg/beam/core/runtime/metricsx/urns.go | 170 ++++++
sdks/go/pkg/beam/core/runtime/options.go | 1 +
sdks/go/pkg/beam/core/runtime/xlangx/expand.go | 69 ++-
sdks/go/pkg/beam/core/runtime/xlangx/namespace.go | 2 +-
.../pkg/beam/core/runtime/xlangx/namespace_test.go | 348 +++++------
sdks/go/pkg/beam/core/runtime/xlangx/translate.go | 228 --------
sdks/go/pkg/beam/doc_test.go | 2 +-
sdks/go/pkg/beam/io/textio/sdf_test.go | 2 +-
sdks/go/pkg/beam/options/jobopts/options.go | 3 +
sdks/go/pkg/beam/pipeline.go | 6 +
sdks/go/pkg/beam/runner.go | 6 +-
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 25 +-
sdks/go/pkg/beam/runners/direct/direct.go | 17 +-
sdks/go/pkg/beam/runners/dot/dot.go | 10 +-
sdks/go/pkg/beam/runners/flink/flink.go | 3 +-
sdks/go/pkg/beam/runners/session/session.go | 6 +-
sdks/go/pkg/beam/runners/spark/spark.go | 3 +-
.../beam/runners/universal/runnerlib/execute.go | 48 +-
.../go/pkg/beam/runners/universal/runnerlib/job.go | 3 +
sdks/go/pkg/beam/runners/universal/universal.go | 28 +-
sdks/go/pkg/beam/runners/vet/vet.go | 8 +-
sdks/go/pkg/beam/testing/ptest/ptest.go | 3 +-
sdks/go/pkg/beam/x/beamx/run.go | 8 +
sdks/go/pkg/beam/xlang.go | 59 +-
sdks/go/test/load/build.gradle | 66 +++
sdks/go/test/load/pardo/pardo.go | 112 ++++
sdks/go/test/load/util.go | 227 ++++++++
sdks/java/container/build.gradle | 63 +-
sdks/java/container/common.gradle | 104 ++++
.../java/container/java11/build.gradle | 11 +-
.../java/container/java8/build.gradle | 11 +-
.../org/apache/beam/sdk/io/CountingSource.java | 3 +-
.../sdk/schemas/FieldValueTypeInformation.java | 32 +-
.../apache/beam/sdk/schemas/JavaBeanSchema.java | 27 +-
.../sdk/schemas/annotations/SchemaCaseFormat.java | 73 +++
.../sdk/schemas/annotations/SchemaFieldName.java | 11 +-
.../beam/sdk/schemas/utils/JavaBeanUtils.java | 39 +-
.../apache/beam/sdk/schemas/utils/POJOUtils.java | 5 +-
.../apache/beam/sdk/testing/UsesTestStream.java | 2 +-
.../java/org/apache/beam/sdk/transforms/DoFn.java | 4 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 70 ++-
.../sdk/transforms/reflect/DoFnSignatures.java | 77 +--
.../splittabledofn/RestrictionTracker.java | 23 +-
.../beam/sdk/util/common/ReflectHelpers.java | 60 ++
.../beam/sdk/schemas/AutoValueSchemaTest.java | 57 ++
.../beam/sdk/schemas/JavaBeanSchemaTest.java | 73 +++
.../beam/sdk/schemas/JavaFieldSchemaTest.java | 55 ++
.../beam/sdk/schemas/utils/SchemaTestUtils.java | 22 +
.../beam/sdk/schemas/utils/TestJavaBeans.java | 129 +++++
.../apache/beam/sdk/schemas/utils/TestPOJOs.java | 56 ++
.../beam/sdk/transforms/GroupIntoBatchesTest.java | 82 +++
sdks/java/extensions/sorter/build.gradle | 46 +-
sdks/java/extensions/sql/build.gradle | 44 +-
.../impl/transform/BeamBuiltinAggregations.java | 143 ++---
.../provider/bigquery/BeamBigQuerySqlDialect.java | 11 +
.../extensions/sql/BeamSqlDslAggregationTest.java | 263 ++++-----
.../zetasql/SupportedZetaSqlBuiltinFunctions.java | 4 +-
.../sql/zetasql/ZetaSQLQueryPlanner.java | 8 +
.../translation/ArrayScanColumnRefToUncollect.java | 4 +-
.../ArrayScanLiteralToUncollectConverter.java | 4 +-
.../translation/ArrayScanToJoinConverter.java | 5 +-
.../translation/SqlOperatorMappingTable.java | 1 +
.../zetasql/unnest/BeamZetaSqlUncollectRel.java | 120 ++++
.../zetasql/unnest/BeamZetaSqlUncollectRule.java | 54 ++
.../sql/zetasql/unnest/BeamZetaSqlUnnestRel.java | 164 ++++++
.../sql/zetasql/unnest/BeamZetaSqlUnnestRule.java | 120 ++++
.../sql/zetasql/unnest/ZetaSqlUnnest.java | 140 +++++
.../sql/zetasql/unnest/package-info.java | 8 +-
.../beam/sdk/extensions/sql/zetasql/TestInput.java | 66 ++-
.../sql/zetasql/ZetaSqlDialectSpecTest.java | 137 ++++-
.../extensions/sql/zetasql/ZetaSqlTestBase.java | 8 +-
.../fn/harness/logging/BeamFnLoggingClient.java | 50 +-
.../harness/logging/BeamFnLoggingClientTest.java | 50 +-
.../expansion-service/build.gradle | 37 ++
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 22 +-
.../beam/sdk/io/gcp/spanner/MutationUtils.java | 320 +++++++++++
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 6 +
.../beam/sdk/io/gcp/spanner/SpannerConfig.java | 8 +
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 112 +++-
.../io/gcp/spanner/SpannerTransformRegistrar.java | 374 ++++++++++++
.../beam/sdk/io/gcp/spanner/StructUtils.java | 387 +++++++++++++
.../beam/sdk/io/gcp/spanner/MutationUtilsTest.java | 285 +++++++++
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 5 +-
.../beam/sdk/io/gcp/spanner/StructUtilsTest.java | 258 +++++++++
sdks/java/io/hadoop-common/build.gradle | 50 +-
sdks/java/io/hadoop-file-system/build.gradle | 57 +-
sdks/java/io/hadoop-format/build.gradle | 104 +++-
.../io/hadoop/format/HadoopFormatIOElasticIT.java | 20 +-
.../sdk/io/hadoop/format/HadoopFormatIOIT.java | 58 +-
.../hadoop/format/HadoopFormatIOTestOptions.java | 6 +
sdks/java/io/hbase/build.gradle | 2 +-
sdks/java/io/hcatalog/build.gradle | 42 +-
sdks/java/io/parquet/build.gradle | 43 +-
sdks/java/maven-archetypes/examples/build.gradle | 1 +
sdks/java/maven-archetypes/examples/sample.txt | 102 ++++
.../src/main/resources/archetype-resources/pom.xml | 8 +-
.../beam/sdk/nexmark/NexmarkConfiguration.java | 2 +-
.../apache/beam/sdk/nexmark/NexmarkLauncher.java | 4 +-
.../apache/beam/sdk/nexmark/NexmarkOptions.java | 30 +-
.../sdk/nexmark/queries/SessionSideInputJoin.java | 2 +-
sdks/python/apache_beam/dataframe/expressions.py | 92 ++-
sdks/python/apache_beam/dataframe/frame_base.py | 8 +-
sdks/python/apache_beam/dataframe/frames.py | 65 ++-
sdks/python/apache_beam/dataframe/frames_test.py | 91 +--
sdks/python/apache_beam/dataframe/io.py | 1 +
.../apache_beam/dataframe/pandas_doctests_test.py | 3 +-
sdks/python/apache_beam/dataframe/partitionings.py | 26 +
sdks/python/apache_beam/dataframe/transforms.py | 62 +-
.../apache_beam/dataframe/transforms_test.py | 17 +-
.../apache_beam/examples/kafkataxi/README.md | 6 +-
sdks/python/apache_beam/internal/metrics/metric.py | 90 +++
.../apache_beam/internal/metrics/metric_test.py | 41 ++
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 36 +-
.../gcp/resource_identifiers.py} | 34 +-
sdks/python/apache_beam/io/gcp/spanner.py | 635 +++++++++++++++++++++
.../io/gcp/tests/xlang_spannerio_it_test.py | 339 +++++++++++
sdks/python/apache_beam/io/iobase.py | 12 +-
sdks/python/apache_beam/io/mongodbio.py | 166 ++++--
sdks/python/apache_beam/io/mongodbio_it_test.py | 94 ++-
sdks/python/apache_beam/io/mongodbio_test.py | 235 +++++++-
sdks/python/apache_beam/io/parquetio.py | 8 +
sdks/python/apache_beam/io/parquetio_test.py | 4 +
sdks/python/apache_beam/metrics/cells.py | 22 +-
sdks/python/apache_beam/metrics/execution.pxd | 4 +-
sdks/python/apache_beam/metrics/execution.py | 71 ++-
sdks/python/apache_beam/metrics/metric.py | 18 +-
sdks/python/apache_beam/metrics/metric_test.py | 16 +
sdks/python/apache_beam/metrics/metricbase.py | 46 +-
.../python/apache_beam/metrics/monitoring_infos.py | 21 +-
.../apache_beam/metrics/monitoring_infos_test.py | 35 ++
sdks/python/apache_beam/pipeline.py | 12 +-
.../apache_beam/runners/dataflow/internal/names.py | 4 +-
.../runners/interactive/background_caching_job.py | 11 +-
.../interactive/background_caching_job_test.py | 42 +-
.../runners/interactive/interactive_environment.py | 49 +-
.../runners/interactive/interactive_runner.py | 4 +-
.../runners/interactive/interactive_runner_test.py | 3 -
.../runners/interactive/pipeline_fragment.py | 9 +-
.../runners/interactive/pipeline_instrument.py | 29 +-
.../runners/interactive/recording_manager.py | 20 +-
.../runners/interactive/recording_manager_test.py | 61 +-
.../runners/interactive/user_pipeline_tracker.py | 150 +++++
.../interactive/user_pipeline_tracker_test.py | 209 +++++++
.../runners/portability/flink_runner_test.py | 3 +-
.../portability/fn_api_runner/fn_runner_test.py | 81 ++-
.../portability/fn_api_runner/translations.py | 29 +-
.../portability/fn_api_runner/translations_test.py | 39 ++
.../apache_beam/runners/worker/sdk_worker.py | 38 +-
.../apache_beam/runners/worker/sdk_worker_test.py | 60 ++
sdks/python/apache_beam/transforms/core.py | 28 +-
sdks/python/apache_beam/transforms/util.py | 49 +-
sdks/python/apache_beam/transforms/util_test.py | 17 +
sdks/python/apache_beam/utils/sharded_key.py | 2 +-
sdks/python/apache_beam/utils/shared.py | 24 +-
sdks/python/setup.py | 4 +-
sdks/python/test-suites/portable/common.gradle | 6 +-
sdks/python/test-suites/tox/common.gradle | 3 +-
sdks/python/test-suites/tox/py38/build.gradle | 7 +
sdks/python/tox.ini | 21 +
settings.gradle | 4 +
.../content/en/documentation/programming-guide.md | 365 +++++++++++-
.../en/documentation/runtime/environments.md | 18 +-
.../site/content/en/get-started/quickstart-java.md | 20 +-
.../partials/section-menu/en/documentation.html | 10 +
.../images/multi-language-pipelines-diagram.svg | 20 +
216 files changed, 10389 insertions(+), 2166 deletions(-)