You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2021/04/22 03:20:24 UTC
[beam] tag nightly-master updated (a17f735 -> fb1d769)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from a17f735 (commit)
to fb1d769 (commit)
from a17f735 [BEAM-12191] Fix a bug that upload_graph doesn't reduce template file size (#14582)
add ff1e3e8 [BEAM-10029] Spanner IO read and write performance tests (#13857)
add 96c3e49 Add tensorboard-data-server license URL for Python container image. (#14583)
add bd66ccf [BEAM-6597] Use runner capabilities to activate short ids.
add 28ca249 Fix wording.
add 07ffbb1 Merge pull request #14577 [BEAM-6597] Use runner capabilities to activate short ids.
add 7365d6e Populate runtime parameters in SDK worker startup.
add 3d1d885 Comment about runtime value usage.
add a3be723 Merge pull request #14547 Populate runtime parameters in SDK worker startup.
add 3564a88 Libraries-BOM 20.0.0 with gRPC and protobuf
add 7fc7e0a Updating assertions for google-http-client 1.39.2
add f03a076 Declare gax-httpjson as a direct dependency of google-cloud-platform
add 62df129 Merge pull request #14527 : [BEAM-11205] Libraries-BOM 20.0.0 with gRPC and protobuf
add 2e68885 Update Java beam container image
add d14c221 Merge pull request #14587: Update Java beam container image
add 077f4ca Add --sdk_container_image Python WorkerOption (#14557)
add 2012180 [BEAM-2085] Add capability to declare resource hints in Python pipeline transforms. (#14390)
add 9ab200c [BEAM-12174] Samza Portable Runner Support (#14554)
add 19e662d Update stringSlice to impl pkg.go.dev/flag#Getter.Get
add 6cdf356 Merge pull request #14595: Update stringSlice to impl pkg.go.dev/flag#Getter.Get
add cc5c07d [BEAM-12197] TPC-DS: Fix SQL-queries syntax
add b3d50ee Merge pull request #1458: [BEAM-12197] TPC-DS: Fix SQL-queries syntax
add 09b4d6a [BEAM-8357] Upgrading auto-value to 1.8 from 1.7.4
add 2ac2fc2 Merge pull request #14581 : [BEAM-8357] Upgrading auto-value to 1.8 from 1.7.4
add b42d8c5 Build source release zip from RC tag
add 46a0b39 Merge pull request #14589: Build source release zip from RC tag
add 75bc72c [BEAM-12009] Copy CalcRelSplitter.
add 1c02ebe [BEAM-12009] Implement Calc splitting rule.
add 93c1744 [BEAM-12009] Use different method signature to avoid spurious null check fail.
add 2c9f744 [BEAM-12009] Move CalcRelSplitter to rel subpackage.
add 387d898 [BEAM-12009] Reduce code duplication between rules by re-implementing existing rules with Calc splitter.
add 0771c40 [BEAM-12009] Only match in BeamCalcSplittingRule if ≥ 1 of its component rel types can implement each expression.
add 1da777a Merge pull request #14476 from ibzib/calc-splitting
add f4fcccc BEAM-12166:Beam Sql - Combine Accumulator return Map fails with class cast exception
add fc873f0 Merge pull request #14534 from anupd22/BEAM-12166
add ce832d4 [BEAM-12194] Enable SqlTransform::registerUdaf in ZetaSQL.
add d6a3256 [BEAM-12194] Code style changes from review.
add c0c95b5 Merge pull request #14588 from ibzib/BEAM-12194
add 1fdfef5 [BEAM-12114] Dataflow should apply KAFKA_READ_OVERRIDE when it's not with runner_v2
add 4439170 Merge pull request #14608 from [BEAM-12114] Dataflow should apply KAFKA_READ_OVERRIDE when it's not with runner_v2
add c1e5b35 Add a BQ option for configuring buffering duration when auto-sharding is used.
add 9182d8d Merge pull request #14590 from [BEAM-11772, BEAM-11408] Add a BQ option for configuring buffering duration when auto-sharding is used
add 9eb6eb5 Make sdk/worker_harness_container_image fully backwards compatible (#14599)
add a46590b [BEAM-11010] Upgrading google-cloud-pubsublite to 0.13.2
add 33aa436 [BEAM-11010] Copying SubscriberOptions from pubsublite repo
add facd2c0 [BEAM-11010] Declaring flogger-system-backend to avoid conflicts
add fb1d769 Merge pull request #14578: [BEAM-11010] Upgrading google-cloud-pubsublite to 0.13.2
No new revisions were added by this update.
Summary of changes:
.test-infra/jenkins/README.md | 2 +
.../job_PerformanceTests_SpannerIO_Python.groovy | 108 +++
CHANGES.md | 1 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 14 +-
.../pipeline/src/main/proto/beam_runner_api.proto | 21 +
.../src/main/scripts/build_release_candidate.sh | 16 +-
runners/google-cloud-dataflow-java/build.gradle | 4 +-
.../beam/runners/dataflow/DataflowRunner.java | 2 +-
runners/samza/build.gradle | 2 +-
.../beam/runners/samza/SamzaExecutionContext.java | 117 +--
.../apache/beam/runners/samza/SamzaJobInvoker.java | 82 ++
.../beam/runners/samza/SamzaJobServerDriver.java | 162 ++--
.../beam/runners/samza/SamzaPipelineRunner.java | 2 +-
.../org/apache/beam/runners/samza/SamzaRunner.java | 5 +-
.../apache/beam/runners/samza/runtime/DoFnOp.java | 9 +-
.../SamzaExecutableStageContextFactory.java | 61 ++
.../samza/translation/ImpulseTranslator.java | 6 +-
.../translation/ParDoBoundMultiTranslator.java | 2 +
.../translation/PortableTranslationContext.java | 9 +-
.../samza/util/SamzaPipelineTranslatorUtils.java | 2 +-
sdks/go/pkg/beam/options/jobopts/stringSlice.go | 5 +
.../container/license_scripts/dep_urls_java.yaml | 2 +-
.../beam/sdk/extensions/gcp/util/GcsUtilTest.java | 19 +-
...LatencyRecordingHttpRequestInitializerTest.java | 4 +-
.../gcp/util/RetryHttpRequestInitializerTest.java | 74 +-
.../extensions/sql/impl/rel/CalcRelSplitter.java | 898 +++++++++++++++++++++
.../sql/impl/rule/BeamCalcSplittingRule.java | 71 ++
.../extensions/sql/impl/utils/CalciteUtils.java | 21 +-
.../sdk/extensions/sql/BeamSqlDslUdfUdafTest.java | 114 +++
.../extensions/sql/zetasql/BeamCalcRelType.java | 157 ++++
.../sql/zetasql/BeamJavaUdfCalcRule.java | 160 +---
.../sql/zetasql/BeamZetaSqlCalcRule.java | 65 +-
.../sql/zetasql/BeamZetaSqlCalcSplittingRule.java | 44 +
.../extensions/sql/zetasql/BeamZetaSqlCatalog.java | 50 +-
...etaSqlCalcRule.java => BeamZetaSqlRelType.java} | 86 +-
.../translation/AggregateScanConverter.java | 49 +-
.../translation/UserFunctionDefinitions.java | 9 +-
.../sql/zetasql/BeamJavaUdfCalcRuleTest.java | 84 ++
.../extensions/sql/zetasql/ZetaSqlJavaUdfTest.java | 72 +-
.../sql/zetasql/ZetaSqlJavaUdfTypeTest.java | 11 +-
.../java/org/apache/beam/fn/harness/FnHarness.java | 1 +
.../fn/harness/control/ProcessBundleHandler.java | 10 +-
.../harness/control/ProcessBundleHandlerTest.java | 12 +
sdks/java/io/google-cloud-platform/build.gradle | 4 +
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 7 +-
.../sdk/io/gcp/bigquery/BatchedStreamingWrite.java | 10 +-
.../beam/sdk/io/gcp/bigquery/BigQueryOptions.java | 8 +
.../beam/sdk/io/gcp/pubsublite/PublisherCache.java | 10 +-
.../sdk/io/gcp/pubsublite/PublisherOptions.java | 4 +-
.../sdk/io/gcp/pubsublite/PublisherOrError.java | 6 +-
.../beam/sdk/io/gcp/pubsublite/Publishers.java | 27 +-
.../beam/sdk/io/gcp/pubsublite/PubsubLiteSink.java | 10 +-
.../sdk/io/gcp/pubsublite/SubscribeTransform.java | 7 +-
.../sdk/io/gcp/pubsublite/SubscriberOptions.java | 65 +-
.../apache/beam/sdk/io/gcp/pubsublite/Uuid.java | 6 +-
.../io/gcp/bigquery/BigQueryServicesImplTest.java | 564 ++++++++-----
.../sdk/io/gcp/pubsublite/PubsubLiteSinkTest.java | 24 +-
.../apache/beam/sdk/tpcds/BeamSqlEnvRunner.java | 15 +-
.../apache/beam/sdk/tpcds/SqlTransformRunner.java | 17 +-
.../beam/sdk/tpcds/TpcdsParametersReader.java | 78 +-
.../tpcds/src/main/resources/queries/query1.sql | 2 +-
.../tpcds/src/main/resources/queries/query12.sql | 30 +-
.../tpcds/src/main/resources/queries/query14.sql | 223 -----
.../tpcds/src/main/resources/queries/query14a.sql | 95 +++
.../tpcds/src/main/resources/queries/query14b.sql | 110 +++
.../tpcds/src/main/resources/queries/query16.sql | 18 +-
.../tpcds/src/main/resources/queries/query20.sql | 23 +-
.../tpcds/src/main/resources/queries/query21.sql | 14 +-
.../tpcds/src/main/resources/queries/query23a.sql | 65 ++
.../queries/{query23.sql => query23b.sql} | 81 +-
.../tpcds/src/main/resources/queries/query24.sql | 119 ---
.../tpcds/src/main/resources/queries/query24a.sql | 35 +
.../tpcds/src/main/resources/queries/query24b.sql | 35 +
.../tpcds/src/main/resources/queries/query32.sql | 40 +-
.../tpcds/src/main/resources/queries/query37.sql | 2 +-
.../queries/{query39.sql => query39a.sql} | 30 +-
.../queries/{query39.sql => query39b.sql} | 29 +-
.../tpcds/src/main/resources/queries/query40.sql | 16 +-
.../tpcds/src/main/resources/queries/query47.sql | 8 +-
.../tpcds/src/main/resources/queries/query5.sql | 26 +-
.../tpcds/src/main/resources/queries/query50.sql | 16 +-
.../tpcds/src/main/resources/queries/query58.sql | 18 +-
.../tpcds/src/main/resources/queries/query59.sql | 8 +-
.../tpcds/src/main/resources/queries/query62.sql | 16 +-
.../tpcds/src/main/resources/queries/query66.sql | 104 +--
.../tpcds/src/main/resources/queries/query72.sql | 10 +-
.../tpcds/src/main/resources/queries/query74.sql | 16 +-
.../tpcds/src/main/resources/queries/query75.sql | 84 +-
.../tpcds/src/main/resources/queries/query77.sql | 47 +-
.../tpcds/src/main/resources/queries/query80.sql | 26 +-
.../tpcds/src/main/resources/queries/query82.sql | 2 +-
.../tpcds/src/main/resources/queries/query90.sql | 4 +-
.../tpcds/src/main/resources/queries/query92.sql | 40 +-
.../tpcds/src/main/resources/queries/query94.sql | 12 +-
.../tpcds/src/main/resources/queries/query95.sql | 14 +-
.../tpcds/src/main/resources/queries/query98.sql | 29 +-
.../tpcds/src/main/resources/queries/query99.sql | 16 +-
.../tpcds/src/main/resources/schemas/store.json | 8 +-
.../beam/sdk/tpcds/TpcdsParametersReaderTest.java | 26 +-
.../gcp/experimental/spannerio_read_perf_test.py | 158 ++++
.../gcp/experimental/spannerio_write_perf_test.py | 146 ++++
.../python/apache_beam/options/pipeline_options.py | 30 +-
.../options/pipeline_options_validator.py | 22 +
.../options/pipeline_options_validator_test.py | 48 ++
sdks/python/apache_beam/pipeline.py | 38 +-
sdks/python/apache_beam/pipeline_test.py | 203 +++++
sdks/python/apache_beam/portability/common_urns.py | 2 +
.../runners/dataflow/dataflow_runner.py | 16 +-
.../runners/dataflow/dataflow_runner_test.py | 54 +-
.../runners/dataflow/internal/apiclient.py | 4 +-
.../runners/dataflow/internal/apiclient_test.py | 60 +-
.../apache_beam/runners/dataflow/internal/names.py | 1 +
.../python/apache_beam/runners/pipeline_context.py | 57 +-
.../apache_beam/runners/pipeline_context_test.py | 32 +-
.../apache_beam/runners/worker/sdk_worker_main.py | 116 +--
.../runners/worker/sdk_worker_main_test.py | 11 +
sdks/python/apache_beam/transforms/core.py | 2 +-
sdks/python/apache_beam/transforms/environments.py | 235 ++++--
.../apache_beam/transforms/environments_test.py | 11 +
sdks/python/apache_beam/transforms/ptransform.py | 30 +
.../apache_beam/transforms/ptransform_test.py | 12 +
sdks/python/apache_beam/transforms/resources.py | 216 +++++
.../apache_beam/transforms/resources_test.py | 66 ++
.../container/license_scripts/dep_urls_py.yaml | 2 +
sdks/python/container/run_validatescontainer.sh | 2 +-
.../en/documentation/runtime/environments.md | 2 +-
126 files changed, 4709 insertions(+), 1871 deletions(-)
create mode 100644 .test-infra/jenkins/job_PerformanceTests_SpannerIO_Python.groovy
create mode 100644 runners/samza/src/main/java/org/apache/beam/runners/samza/SamzaJobInvoker.java
create mode 100644 runners/samza/src/main/java/org/apache/beam/runners/samza/runtime/SamzaExecutableStageContextFactory.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rel/CalcRelSplitter.java
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/rule/BeamCalcSplittingRule.java
create mode 100644 sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamCalcRelType.java
create mode 100644 sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCalcSplittingRule.java
copy sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/{BeamZetaSqlCalcRule.java => BeamZetaSqlRelType.java} (52%)
create mode 100644 sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamJavaUdfCalcRuleTest.java
delete mode 100644 sdks/java/testing/tpcds/src/main/resources/queries/query14.sql
create mode 100644 sdks/java/testing/tpcds/src/main/resources/queries/query14a.sql
create mode 100644 sdks/java/testing/tpcds/src/main/resources/queries/query14b.sql
create mode 100644 sdks/java/testing/tpcds/src/main/resources/queries/query23a.sql
rename sdks/java/testing/tpcds/src/main/resources/queries/{query23.sql => query23b.sql} (50%)
delete mode 100644 sdks/java/testing/tpcds/src/main/resources/queries/query24.sql
create mode 100644 sdks/java/testing/tpcds/src/main/resources/queries/query24a.sql
create mode 100644 sdks/java/testing/tpcds/src/main/resources/queries/query24b.sql
copy sdks/java/testing/tpcds/src/main/resources/queries/{query39.sql => query39a.sql} (61%)
rename sdks/java/testing/tpcds/src/main/resources/queries/{query39.sql => query39b.sql} (61%)
create mode 100644 sdks/python/apache_beam/io/gcp/experimental/spannerio_read_perf_test.py
create mode 100644 sdks/python/apache_beam/io/gcp/experimental/spannerio_write_perf_test.py
create mode 100644 sdks/python/apache_beam/transforms/resources.py
create mode 100644 sdks/python/apache_beam/transforms/resources_test.py