You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2022/06/15 04:12:11 UTC
[beam] branch nightly-refs/heads/master updated (b426d34f622 -> ac20321008e)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from b426d34f622 [BEAM-14557] Read and Seek Runner Capabilities in Go SDK (#17821)
add 080f54a144a [BEAM-13806] Add x-lang BigQuery IO integration test to Go SDK. (#16818)
add 6e450a92da7 [BEAM-14265] Add watermark hold for all timers
add 7bf822a5419 Merge pull request #17809: [BEAM-14265] Add watermark hold for all timers
add 12ba4cea9d6 Bump Python beam-master container (#21820)
add a94feb146b7 Split PytorchModelHandler into PytorchModelHandlerTensor and PytorchModelHandlerKeyedTensor (#21810)
add ab8977f4299 Fix Hadoop Downloader Range not correct (#21778)
add 6cfcc9d98bd [BEAM-14036] Read Configuration for Pub/Sub SchemaTransform (#17730)
add 091c05ce4b6 [Go SDK] Add more info to Worker Status API (#21776)
add a2c3ae60966 Make PeriodicImpulse generates unbounded PCollection (#21815)
add 8c94fed2a03 [BEAM-14267] Update watchForNewFiles to allow watching updated files (#17305)
add 94e240052aa fix timestamp conversion in Google Cloud Datastore Connector (#17789)
add b91b16fcd8e Update references to Jira to GH for the Go label (#21830)
add b527ac4bcac [#21853] Adjust Go cross-compile to target entire package (#21854)
add ecea6de6dc8 [BEAM-14553] Add destination coder to FileResultCoder components (#17818)
add 272b853272f Add transform names to help debug flaky test
add 322a4bc29e5 Merge pull request #21745 from nielm/flakyIntegrationTest
add 4b33a38cdd0 copyedited README for RunInference examples (#21855)
add efbce542728 Automatically enable Runner v2 for pipelines that use cross-language transforms
add e7c021d4508 Merge pull request #21788: Automatically enable Runner v2 for pipelines that use cross-language transforms.
add 5f04b97fa5e Document and test overriding batch type inference (#21844)
add 8a1aad68243 Update references to Jira to GH for the Python SDK (#21831)
add f7896d819c5 Adjust Jenkins configuration to allow more memory per JVM
add 18ab78c8c42 Merge pull request #21858: Adjust Jenkins configuration to allow more memory per JVM (fixes #20819)
add ac20321008e add highlights to changes (#21865)
No new revisions were added by this update.
Summary of changes:
.test-infra/jenkins/CommonJobProperties.groovy | 7 +-
CHANGES.md | 2 +
.../beam/runners/core/construction/External.java | 8 +-
.../beam/runners/core/StatefulDoFnRunner.java | 21 +-
.../wrappers/streaming/DoFnOperator.java | 109 +++------
.../streaming/state/FlinkStateInternals.java | 18 +-
.../wrappers/streaming/DoFnOperatorTest.java | 28 +--
runners/google-cloud-dataflow-java/build.gradle | 4 +-
.../beam/runners/dataflow/DataflowRunner.java | 43 +++-
.../beam/runners/dataflow/DataflowRunnerTest.java | 64 +++++
sdks/go/container/boot.go | 4 +-
sdks/go/pkg/beam/artifact/materialize.go | 4 +-
sdks/go/pkg/beam/core/graph/coder/coder.go | 4 +-
sdks/go/pkg/beam/core/graph/coder/row.go | 2 +-
sdks/go/pkg/beam/core/graph/coder/row_test.go | 2 +-
sdks/go/pkg/beam/core/graph/mtime/time.go | 2 +-
sdks/go/pkg/beam/core/metrics/store.go | 50 ++++
sdks/go/pkg/beam/core/runtime/exec/coder_test.go | 2 +-
sdks/go/pkg/beam/core/runtime/exec/cogbk.go | 2 +-
sdks/go/pkg/beam/core/runtime/exec/combine.go | 2 +-
sdks/go/pkg/beam/core/runtime/exec/datasource.go | 4 +-
.../go/pkg/beam/core/runtime/exec/dynsplit_test.go | 2 +-
sdks/go/pkg/beam/core/runtime/exec/sdf.go | 4 +-
sdks/go/pkg/beam/core/runtime/exec/translate.go | 2 +-
sdks/go/pkg/beam/core/runtime/graphx/coder.go | 8 +-
sdks/go/pkg/beam/core/runtime/graphx/cogbk.go | 2 +-
sdks/go/pkg/beam/core/runtime/graphx/dataflow.go | 4 +-
sdks/go/pkg/beam/core/runtime/graphx/translate.go | 6 +-
sdks/go/pkg/beam/core/runtime/harness/datamgr.go | 2 +-
sdks/go/pkg/beam/core/runtime/harness/harness.go | 52 ++--
sdks/go/pkg/beam/core/runtime/harness/init/init.go | 3 +-
.../go/pkg/beam/core/runtime/harness/monitoring.go | 2 +-
.../core/runtime/harness/statecache/statecache.go | 7 +
.../pkg/beam/core/runtime/harness/worker_status.go | 66 ++++-
.../core/runtime/harness/worker_status_test.go | 6 +-
sdks/go/pkg/beam/core/util/jsonx/jsonx.go | 2 +-
sdks/go/pkg/beam/io/textio/textio.go | 2 +-
sdks/go/pkg/beam/io/xlang/bigqueryio/bigquery.go | 7 +
sdks/go/pkg/beam/io/xlang/kafkaio/kafka.go | 2 +-
.../beam/model/pipeline_v1/beam_runner_api.pb.go | 6 +-
sdks/go/pkg/beam/pardo.go | 4 +-
sdks/go/pkg/beam/pardo_test.go | 2 +-
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 2 +-
.../beam/runners/universal/runnerlib/compile.go | 4 +-
.../pkg/beam/runners/universal/runnerlib/stage.go | 2 +-
sdks/go/pkg/beam/runners/vet/vet.go | 4 +-
.../go/pkg/beam/transforms/stats/quantiles_test.go | 2 +-
sdks/go/test/build.gradle | 4 +
sdks/go/test/integration/flags.go | 5 +
sdks/go/test/integration/integration.go | 25 +-
.../integration/io/xlang/bigquery/bigquery.go} | 5 +-
.../integration/io/xlang/bigquery/bigquery_test.go | 270 +++++++++++++++++++++
.../go/test/integration/io/xlang/bigquery/table.go | 52 ++++
sdks/go/test/integration/wordcount/wordcount.go | 3 +-
sdks/go/test/integration/xlang/expansion_test.go | 2 +-
.../go/test/regression/coders/fromyaml/fromyaml.go | 4 +-
.../main/java/org/apache/beam/sdk/io/AvroIO.java | 34 ++-
.../java/org/apache/beam/sdk/io/FileBasedSink.java | 7 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 115 +++++++--
.../main/java/org/apache/beam/sdk/io/TextIO.java | 31 ++-
.../java/org/apache/beam/sdk/io/WriteFiles.java | 6 +-
.../java/org/apache/beam/sdk/io/FileIOTest.java | 65 ++++-
.../beam/sdk/io/gcp/datastore/EntityToRow.java | 3 +-
.../PubsubSchemaTransformReadConfiguration.java | 171 +++++++++++++
.../gcp/datastore/EntityToRowRowToEntityTest.java | 3 +-
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 12 +-
.../apache/beam/sdk/io/gcp/storage/GcsMatchIT.java | 152 ++++++++++++
sdks/python/.pylintrc | 4 +-
sdks/python/apache_beam/coders/coder_impl.py | 26 +-
sdks/python/apache_beam/coders/coders.py | 7 +-
sdks/python/apache_beam/coders/row_coder.py | 3 +-
sdks/python/apache_beam/coders/row_coder_test.py | 7 +-
sdks/python/apache_beam/coders/typecoders_test.py | 2 +-
sdks/python/apache_beam/dataframe/frame_base.py | 8 +-
sdks/python/apache_beam/dataframe/frames.py | 113 +++++----
sdks/python/apache_beam/dataframe/frames_test.py | 72 +++---
sdks/python/apache_beam/dataframe/io.py | 4 +-
sdks/python/apache_beam/dataframe/io_test.py | 4 +-
.../apache_beam/dataframe/pandas_doctests_test.py | 23 +-
sdks/python/apache_beam/dataframe/schemas_test.py | 5 +-
.../examples/complete/game/game_stats_it_test.py | 4 +-
.../examples/complete/game/leader_board_it_test.py | 4 +-
.../examples/complete/juliaset/setup.py | 6 +-
.../examples/dataframe/taxiride_test.py | 3 +-
.../apache_beam/examples/inference/README.md | 54 +++--
.../transforms/aggregation/combineglobally_test.py | 4 +-
.../examples/streaming_wordcount_debugging.py | 3 +-
.../streaming_wordcount_debugging_it_test.py | 3 +-
sdks/python/apache_beam/io/external/gcp/pubsub.py | 4 +-
.../io/external/xlang_jdbcio_it_test.py | 6 +-
.../io/external/xlang_kinesisio_it_test.py | 6 +-
sdks/python/apache_beam/io/fileio.py | 2 +-
sdks/python/apache_beam/io/fileio_test.py | 9 +-
.../io/gcp/big_query_query_to_table_it_test.py | 5 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 17 +-
.../apache_beam/io/gcp/bigquery_file_loads.py | 7 +-
.../apache_beam/io/gcp/bigquery_read_it_test.py | 3 +-
sdks/python/apache_beam/io/gcp/bigtableio.py | 4 +-
sdks/python/apache_beam/io/gcp/bigtableio_test.py | 3 +-
.../apache_beam/io/gcp/gcsio_integration_test.py | 6 +-
sdks/python/apache_beam/io/gcp/pubsub.py | 4 +-
.../apache_beam/io/gcp/pubsub_integration_test.py | 7 +-
sdks/python/apache_beam/io/gcp/pubsub_test.py | 3 +-
sdks/python/apache_beam/io/hadoopfilesystem.py | 5 +-
.../python/apache_beam/io/hadoopfilesystem_test.py | 25 +-
sdks/python/apache_beam/io/watermark_estimators.py | 4 +-
sdks/python/apache_beam/ml/inference/base.py | 17 +-
.../apache_beam/ml/inference/pytorch_inference.py | 162 +++++++++----
.../ml/inference/pytorch_inference_test.py | 23 +-
.../ml/inference/sklearn_inference_test.py | 24 +-
.../python/apache_beam/options/pipeline_options.py | 5 +-
.../apache_beam/options/pipeline_options_test.py | 9 +-
.../apache_beam/options/value_provider_test.py | 6 +-
sdks/python/apache_beam/pipeline.py | 17 +-
sdks/python/apache_beam/pvalue.py | 8 +-
sdks/python/apache_beam/runners/common.py | 36 ++-
sdks/python/apache_beam/runners/common_test.py | 18 ++
.../runners/dataflow/dataflow_metrics.py | 7 +-
.../runners/dataflow/dataflow_metrics_test.py | 4 +-
.../runners/dataflow/dataflow_runner.py | 17 +-
.../runners/dataflow/dataflow_runner_test.py | 10 +-
.../runners/dataflow/internal/apiclient.py | 5 +-
.../apache_beam/runners/dataflow/internal/names.py | 4 +-
.../runners/dataflow/test_dataflow_runner.py | 4 +-
.../apache_beam/runners/direct/direct_runner.py | 3 +-
.../runners/direct/direct_runner_test.py | 4 +-
.../runners/direct/transform_evaluator.py | 5 +-
.../runners/interactive/augmented_pipeline.py | 9 +-
.../runners/interactive/caching/streaming_cache.py | 6 +-
.../dataproc/dataproc_cluster_manager.py | 5 +-
.../runners/interactive/interactive_beam.py | 5 +-
.../runners/interactive/interactive_runner.py | 3 +-
.../runners/interactive/recording_manager.py | 4 +-
.../apache_beam/runners/interactive/utils.py | 4 +-
.../python/apache_beam/runners/pipeline_context.py | 6 +-
.../runners/portability/flink_runner_test.py | 17 +-
.../portability/fn_api_runner/fn_runner_test.py | 45 +++-
.../portability/fn_api_runner/translations.py | 6 +-
.../portability/fn_api_runner/worker_handlers.py | 3 +-
.../runners/portability/local_job_service.py | 4 +-
.../runners/portability/portable_runner.py | 7 +-
.../runners/portability/portable_runner_test.py | 10 +-
.../runners/portability/samza_runner_test.py | 16 +-
.../runners/portability/spark_runner_test.py | 20 +-
.../portability/spark_uber_jar_job_server.py | 4 +-
.../apache_beam/runners/portability/stager_test.py | 11 +-
sdks/python/apache_beam/runners/sdf_utils.py | 2 +-
.../apache_beam/runners/worker/bundle_processor.py | 3 +-
.../apache_beam/runners/worker/data_plane.py | 6 +-
.../apache_beam/runners/worker/operations.py | 4 +-
.../apache_beam/runners/worker/sdk_worker_main.py | 3 +-
.../testing/benchmarks/nexmark/queries/query10.py | 5 +-
.../testing/benchmarks/nexmark/queries/query4.py | 3 +-
.../testing/benchmarks/nexmark/queries/query5.py | 3 +-
.../python/apache_beam/testing/test_stream_test.py | 21 +-
.../apache_beam/tools/coders_microbenchmark.py | 3 +-
.../apache_beam/transforms/batch_dofn_test.py | 34 ++-
.../transforms/combinefn_lifecycle_test.py | 2 +-
sdks/python/apache_beam/transforms/combiners.py | 10 +-
sdks/python/apache_beam/transforms/core.py | 80 ++++--
.../apache_beam/transforms/periodicsequence.py | 3 +
.../transforms/periodicsequence_test.py | 4 +-
sdks/python/apache_beam/transforms/ptransform.py | 2 +-
.../apache_beam/transforms/ptransform_test.py | 3 +-
.../apache_beam/transforms/sideinputs_test.py | 12 +-
sdks/python/apache_beam/transforms/stats.py | 3 +-
sdks/python/apache_beam/transforms/trigger.py | 14 +-
sdks/python/apache_beam/transforms/trigger_test.py | 8 +-
.../apache_beam/transforms/userstate_test.py | 4 +-
sdks/python/apache_beam/transforms/util.py | 6 +-
sdks/python/apache_beam/typehints/batch.py | 5 +-
.../typehints/native_type_compatibility.py | 8 +-
.../typehints/native_type_compatibility_test.py | 4 +-
.../apache_beam/typehints/typed_pipeline_test.py | 21 +-
sdks/python/apache_beam/typehints/typehints.py | 9 +-
sdks/python/apache_beam/utils/counters.py | 3 +-
sdks/python/apache_beam/utils/retry.py | 4 +-
sdks/python/apache_beam/utils/timestamp.py | 12 +-
sdks/python/apache_beam/utils/timestamp_test.py | 6 +-
sdks/python/apache_beam/utils/urns.py | 6 +-
sdks/python/build-requirements.txt | 2 +-
sdks/python/mypy.ini | 2 +-
sdks/python/scripts/run_integration_test.sh | 2 +-
sdks/python/test-suites/dataflow/common.gradle | 2 +-
sdks/python/test-suites/direct/xlang/build.gradle | 2 +-
sdks/python/test-suites/portable/common.gradle | 2 +-
sdks/python/test-suites/tox/py37/build.gradle | 2 +-
sdks/python/test-suites/tox/py38/build.gradle | 2 +-
sdks/python/test-suites/tox/py39/build.gradle | 2 +-
sdks/python/test-suites/tox/pycommon/build.gradle | 2 +-
sdks/python/tox.ini | 6 +-
191 files changed, 2170 insertions(+), 780 deletions(-)
copy sdks/go/{pkg/beam/core/runtime/harness/gen.go => test/integration/io/xlang/bigquery/bigquery.go} (88%)
create mode 100644 sdks/go/test/integration/io/xlang/bigquery/bigquery_test.go
create mode 100644 sdks/go/test/integration/io/xlang/bigquery/table.go
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaTransformReadConfiguration.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsMatchIT.java