You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2022/10/23 05:05:11 UTC
[beam] branch nightly-refs/heads/master updated (948d9e2d558 -> 8df6f67c65b)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 948d9e2d558 Blog post for Hop web in Google Cloud (#23652)
add 1920b881975 [Playground] Examples CI (#23476)
add 143dc6fd6f0 [Tour Of Beam] README update (#23318)
add c2aa2286642 [Jenkins,Spark] Stop running Nexmark suite for deprecated Spark 2 runner and ensure consistent version of Spark is used for Nexmark tests (fixes #23634)
add 690f3517def Merge pull request #23639: [Jenkins,Spark] Stop running Nexmark suite for deprecated Spark 2 runners
add c5a2f5fbdfa Bump google.golang.org/api from 0.98.0 to 0.99.0 in /sdks (#23655)
add 0bed47b288b Fix beam_PerformanceTests_PubsubIOIT_Python_Streaming (#23607)
add 2f4ea0f37a6 2.42.0 Release Blog Post (#23406)
add 3b6cd835903 Docs for state in go (#22965)
add 02b6a3e04a3 Fix typo in 2.42.0 blog.
add 693725d6017 Adjust 2.42.0 publishing time.
add 10e15a96383 Adds a Java RunInference example (#23619)
add 34003c6cb7f Fixes #23627: Speed up website checks (#23673)
add a6e25ae421c Suppress a FloatingPointLiteralPrecision error (#23667)
add c66e5f2c982 Improved test coverage and fix the implementation of Inject and CoGBK (#23307)
add 061337b2f43 Fix python log_level_overrides cannot be used on flink and other portable runners (#23668)
add 00e55256922 Better error for disabling runner v2 with cross language pipelines. (#23669)
add 2b4bb6b683f Update scopes to match the existing set and update test to clear credentials during failure (#23677)
add df80a0599c4 [TPC-DS] Use "nonpartitioned" input for Jenkins jobs
add fe41855c322 Merge pull request #23665: [TPC-DS] Use "nonpartitioned" input for Jenkins jobs
add 5d735aef207 [GitHub Actions] - Run RC Validations Workflow (#23531)
add 5cd7dd74c84 Add workflow to update milestone on issue close (#23629)
add 107a43d27ea add website page about data processing for ML (#23552)
add 78e1c0a0184 [Go SDK] Dataframe API wrapper (#23450)
add fac0d03fd56 [Go SDK]: Adds Automated Python Expansion Service (#23582)
add 116327c52a7 Include CombineFn's in __all__ (#23685)
add bd8110c765d Bump google.golang.org/grpc from 1.50.0 to 1.50.1 in /sdks (#23654)
add 2e2bb66f82a [Playground][Frontend] Tags filter for Examples Catalog (#22074) (#23532)
add 2ad17b0dd20 [Go SDK] Extract output coders in expandCrossLanguage (#23641)
add 61f7bfe38ee Python 3.10 support (#23587)
add c087bb1258e Fixes #22192: Avoids nullpointer error. Preserves previous behavior. (#23679)
add 423076416e9 Migrate nexmark to common config for cron jobs
add 27bff7d1aed Merge pull request #23609: Migrate nexmark to common config for cron jobs
add fef5ea910aa Deflaking tests for BQ row insertions. These tests were flaky due to race conditions against expected results (#23678)
add 35ebad91ffa beam-perf
add 45ff95c24ae fix
add aaef80bb6ef Merge pull request #23680 from kileys/fixit
add 1ac50765f86 Add java 11 home to jenkins test (#23708)
add 9f40bbf9c3b enable automatic expansion service (#23699)
add a78c7d7dec9 Remove numpy C API dep from public declarations.
add ff3574fa022 Merge pull request #23649 Remove numpy C API dep from public declarations.
add ae05c25854f add expansion service option (#23712)
add bf9f61037ce Downgrade container cryptography version to avoid yanked version (#23711)
add 2d027d335a5 Update portable runner test timeout (#23696)
add 171d3e3d9d7 Merge pull request #23510: Vortex multiplexing streams
add 10977bfd34e Io jms fix ack message checkpoint (#22932)
add b5a0f4895cc [Playground] Examples CD (#23664)
add 7b6cc89db07 Keep Spark version in a single place only (BeamModulePlugin)
add 1179fdca1b7 Merge pull request #23603: Keep Spark version in a single place only (BeamModulePlugin)
add 437c015a050 Update release instructions in Python 3.10 (#23702)
add bfeb476f9f5 Move Tensorflow Documentation (#23729)
add 8a63ac9bf12 Bump golang.org/x/text from 0.3.7 to 0.4.0 in /sdks (#23686)
add 3dd20722fe1 Unit Content markdown styles (#23592) (#23662)
add 972e66ca780 Add reopen issue command (#23733)
add 8e8e89ebea8 Add example of real time Anomaly Detection using RunInference (#23497)
add 61b8677a3d6 Support TIMESTAMP type in BigQueryIO with BEAM_ROW output type, and in read_gbq (#23070)
add b4af23d8414 Add PytorchBatchConverter (#23296)
add 259cec036cd Pin version to grpcio in build-requirements.txt (#23735)
add ce4ce756d6b Bump up python container versions. (#23716)
add 72e27f47a78 Reduce log flood in Python PostCommit flink task (#23635)
add 3ab9507c43d Speed up check on website links (#23737)
add 728e8ecc8a4 Remove yeandy from reviewers (#23753)
add 9fc4a5cb9f9 tests: fixes SpannerIO unavailable retry test
add 12a3e4a95b0 Merge pull request #23748: fixes SpannerIO unavailable retry test
add 9ff01a1e505 Revert bigdataoss version upgrade (#23727)
add 37ca2e1f332 Updating Python dependencies for the 2.43.0 release
add 7bedcd76a11 Merge pull request #23743: Updating Python dependencies for the 2.43.0 release
add 4113c6dd45c Moving to 2.44.0-SNAPSHOT on master branch.
add ebbd19d9394 Update the timeout in ValidatesContainer suite. (#23732)
add 40c8dcc05c2 Issue#23599 Updated dataframe notebook
add 49ef118b629 Added a missing line break.
add f0e003bbaa0 Updated dataframe notebook
add 35419247faf Verify that secondary key coder is deterministic in SortValues
add c557b977493 Merge pull request #23742: Verify that secondary key coder is deterministic in SortValues
add 3e8bc10c767 fix lints
add 3d055da2fce Merge pull request #23759 from riteshghorse/import-fix
add a138a4f86f6 Update google cloud vision >= 2.0.0 (#23755)
add 2e49c7efa4e Update GcsIO initialization to support converting input parameters to PipelineOptions for authentication (#23766)
add 69fe1cc86f0 Adds instructions for running the Multi-language Java quickstart from released Beam (#23721)
add 231f07d4af4 Remove Spark2 from Java testing projects (addresses #23728) (#23749)
add cba999a2deb bugfix/wrong-notebook-linl (#23777)
add e4aa86fbbfe [CdapIO] Integration CdapIO with SparkReceiverIO (#22584)
add 40b283b45cc Avoid Circular imports related to bigquery_schema_tools (#23731)
add b48cf30c902 Use Flink 1.13 for load tests (#23767)
add ea566b7f474 Remove obsolete sparkRunner task from hadoop-format: not triggered, no tests selected (addresses #23728)
add a0f4e5e6adb Merge pull request #23773: Remove obsolete sparkRunner task from hadoop-format
add 6cf92145aef Remove obsolete native text io translation. (#23549)
add 15ab5003e21 Eliminate nullness errors from GenerateSequence (#23744)
add 36d9112f9a8 Re-enable PubsubTableProviderIT.testSQLSelectsArrayAttributes
add 00566272a45 Merge pull request #23778: Re-enable PubsubTableProviderIT.testSQLSelectsArrayAttributes
add 2d151c306ee Add logos to case-studies "Also Used By" (#23781)
add 8dd87491dcb Avoid pickling unstable reference to moved proto classes. (#23739)
add 8f300b07399 Allow local packages in requirements.txt dependency list. (#23684)
add aadbe31e975 Unskip test_generated_class_pickle for cloudpickle.
add 77e96da7d41 Merge pull request #23791 Unskip test_generated_class_pickle for cloudpickle.
add 01da3fcb3e3 Revert "Update BQIO to a single scheduled executor service reduce threads (#23234)" (#23793)
add 37763c99b8e Updates Python test expansion service to use Cloud Pickle (#23786)
add d38f577624e Merge pull request #23795: Revert 23234: issue #23794
add 8df6f67c65b Merge pull request #23556: Forward failed storage-api row inserts to the failedStorageApiInserts PCollection addresses #23628
No new revisions were added by this update.
Summary of changes:
.github/ACTIONS.md | 1 +
.github/ISSUE_TEMPLATE/bug.yml | 1 +
.github/ISSUE_TEMPLATE/feature.yml | 3 +-
.github/ISSUE_TEMPLATE/task.yml | 3 +-
.github/REVIEWERS.yml | 1 -
.github/actions/common-rc-validation/action.yaml | 54 ++
.../test-properties.json | 14 +-
.github/workflows/assign_milestone.yml | 84 ++++
.github/workflows/build_wheels.yml | 8 +-
.../workflows/playground_examples_cd.yml | 26 +-
.../workflows/playground_examples_cd_reusable.yml | 76 +++
.github/workflows/playground_examples_ci.yml | 50 ++
.../workflows/playground_examples_ci_reusable.yml | 166 ++++++
.github/workflows/python_tests.yml | 5 +-
.github/workflows/run_rc_validation.yml | 559 +++++++++++++++++++++
.github/workflows/self-assign.yml | 8 +
.github/workflows/tour_of_beam_examples_ci.yml | 47 ++
.test-infra/jenkins/CommonTestProperties.groovy | 8 +-
.test-infra/jenkins/Flink.groovy | 4 +-
.test-infra/jenkins/PythonTestProperties.groovy | 3 +-
.../job_LoadTests_Combine_Flink_Python.groovy | 2 +-
.../jenkins/job_LoadTests_GBK_Flink_Python.groovy | 2 +-
.../job_LoadTests_ParDo_Flink_Python.groovy | 2 +-
.../job_LoadTests_coGBK_Flink_Python.groovy | 2 +-
.../job_PerformanceTests_InfluxDBIO_IT.groovy | 3 +-
.../job_PerformanceTests_PubsubIO_Python.groovy | 2 +-
.../job_PostCommit_Java_Nexmark_Dataflow.groovy | 181 +------
.../job_PostCommit_Java_Nexmark_Dataflow_V2.groovy | 44 +-
...stCommit_Java_Nexmark_Dataflow_V2_Java11.groovy | 44 +-
...stCommit_Java_Nexmark_Dataflow_V2_Java17.groovy | 44 +-
.../job_PostCommit_Java_Nexmark_Direct.groovy | 141 +-----
.../job_PostCommit_Java_Nexmark_Flink.groovy | 133 +----
.../job_PostCommit_Java_Nexmark_Spark.groovy | 195 +------
.../job_PostCommit_Java_Tpcds_Dataflow.groovy | 4 +-
.../jenkins/job_PostCommit_Java_Tpcds_Flink.groovy | 4 +-
.../jenkins/job_PostCommit_Java_Tpcds_Spark.groovy | 8 +-
...Commit_Python_Chicago_Taxi_Example_Flink.groovy | 2 +-
...stCommit_Python_ValidatesRunner_Dataflow.groovy | 1 +
.../jenkins/job_PreCommit_Portable_Python.groovy | 2 +-
.../jenkins/job_Publish_SDK_Image_Snapshots.groovy | 1 +
CHANGES.md | 17 +-
CI.md | 15 +
build.gradle.kts | 14 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 20 +-
examples/java/build.gradle | 1 +
.../multilanguage/PythonDataframeWordCount.java | 1 -
.../multilanguage/SklearnMnistClassification.java | 161 ++++++
examples/multi-language/README.md | 132 ++++-
examples/multi-language/build.gradle | 11 +-
examples/notebooks/tour-of-beam/dataframes.ipynb | 3 +-
gradle.properties | 4 +-
learning/tour-of-beam/backend/README.md | 60 +++
.../frontend/lib/pages/tour/widgets/content.dart | 9 +-
.../lib/pages/tour/widgets/unit_content.dart | 52 +-
learning/tour-of-beam/frontend/pubspec.lock | 2 +-
learning/tour-of-beam/frontend/pubspec.yaml | 1 +
.../examples/components/examples_components.dart | 5 +-
.../modules/examples/components/filter/filter.dart | 97 ++++
.../{category_bubble.dart => tag_bubble.dart} | 16 +-
.../{category_bubble.dart => type_bubble.dart} | 6 +-
.../examples/components/filter/type_filter.dart | 59 ---
.../components/search_field/search_field.dart | 6 +-
.../lib/modules/examples/example_selector.dart | 2 +-
.../playground/states/example_selector_state.dart | 101 ++--
.../lib/src/cache/example_cache.dart | 1 +
.../lib/src/constants/colors.dart | 2 +
.../lib/src/constants/sizes.dart | 1 +
.../example_loaders/content_example_loader.dart | 11 +-
.../example_loaders/empty_example_loader.dart | 1 +
.../lib/src/models/example.dart | 2 +
.../lib/src/models/example_base.dart | 2 +
.../example_client/grpc_example_client.dart | 1 +
.../playground_components/lib/src/theme/theme.dart | 50 ++
.../frontend/playground_components/pubspec.yaml | 1 +
.../test/src/common/categories.dart | 20 +-
.../test/src/common/examples.dart | 7 +-
.../states/example_selector_state_test.dart | 56 ++-
playground/infrastructure/cd_helper.py | 42 +-
playground/infrastructure/checker.py | 104 ++--
playground/infrastructure/ci_helper.py | 8 +-
playground/infrastructure/grpc_client.py | 38 +-
playground/infrastructure/helper.py | 5 +-
playground/infrastructure/requirements.txt | 3 +-
playground/infrastructure/test_checker.py | 47 ++
playground/infrastructure/test_ci_helper.py | 18 +-
playground/infrastructure/test_helper.py | 10 +-
.../python-release/python_release_automation.sh | 2 +-
.../python_release_automation_utils.sh | 6 +-
release/src/main/scripts/mass_comment.py | 1 +
.../beam/runners/flink/FlinkPipelineRunner.java | 16 +-
runners/spark/2/build.gradle | 4 +-
runners/spark/3/build.gradle | 6 +-
sdks/go.mod | 18 +-
sdks/go.sum | 46 +-
sdks/go/examples/snippets/04transforms.go | 157 ++++++
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/go/pkg/beam/core/runtime/graphx/translate.go | 14 +
sdks/go/pkg/beam/core/runtime/xlangx/expand.go | 84 +++-
.../{core.go => runtime/xlangx/expand_test.go} | 36 +-
.../core/runtime/xlangx/expansionx/download.go | 72 ++-
.../beam/core/runtime/xlangx/expansionx/process.go | 21 +-
.../core/runtime/xlangx/expansionx/process_test.go | 4 +-
sdks/go/pkg/beam/core/runtime/xlangx/registry.go | 35 +-
sdks/go/pkg/beam/runners/direct/direct_test.go | 59 +++
sdks/go/pkg/beam/runners/direct/gbk.go | 45 +-
sdks/go/pkg/beam/schema.go | 17 +
.../beam/transforms/xlang/dataframe/dataframe.go | 87 ++++
.../pkg/beam/transforms/xlang/python/external.go | 131 +++++
sdks/go/test/integration/integration.go | 1 +
.../transforms/xlang/dataframe/dataframe.go | 48 ++
.../transforms/xlang/dataframe/dataframe_test.go | 60 +++
sdks/go/test/regression/lperror.go | 2 +-
.../java/org/apache/beam/sdk/io/FileSystems.java | 1 +
.../org/apache/beam/sdk/io/GenerateSequence.java | 30 +-
.../apache/beam/sdk/options/SdkHarnessOptions.java | 62 ++-
.../org/apache/beam/sdk/io/FileSystemsTest.java | 28 ++
.../beam/sdk/extensions/sorter/SortValues.java | 13 +-
.../provider/pubsub/PubsubTableProviderIT.java | 2 -
.../fn/harness/logging/BeamFnLoggingClient.java | 30 +-
sdks/java/io/cdap/build.gradle | 3 +
.../java/org/apache/beam/sdk/io/cdap/CdapIO.java | 260 +++++++++-
.../org/apache/beam/sdk/io/cdap/ConfigWrapper.java | 4 +
.../org/apache/beam/sdk/io/cdap/MappingUtils.java | 112 ++++-
.../java/org/apache/beam/sdk/io/cdap/Plugin.java | 84 ++--
.../java/org/apache/beam/sdk/io/cdap/CdapIOIT.java | 6 +-
.../org/apache/beam/sdk/io/cdap/CdapIOTest.java | 68 ++-
.../apache/beam/sdk/io/cdap/EmployeeConfig.java | 5 +-
.../org/apache/beam/sdk/io/cdap/PluginTest.java | 6 +-
.../sdk/io/cdap/{ => batch}/EmployeeBatchSink.java | 4 +-
.../io/cdap/{ => batch}/EmployeeBatchSource.java | 6 +-
.../io/cdap/{ => batch}/EmployeeInputFormat.java | 3 +-
.../{ => batch}/EmployeeInputFormatProvider.java | 4 +-
.../io/cdap/{ => batch}/EmployeeOutputFormat.java | 9 +-
.../{ => batch}/EmployeeOutputFormatProvider.java | 4 +-
.../sdk/io/cdap/streaming/EmployeeReceiver.java | 91 ++++
.../io/cdap/streaming/EmployeeStreamingSource.java | 76 +++
.../beam/sdk/io/gcp/bigquery/BigQueryOptions.java | 11 +
.../beam/sdk/io/gcp/bigquery/BigQueryServices.java | 4 +-
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 33 +-
.../beam/sdk/io/gcp/bigquery/StorageApiLoads.java | 100 ++--
.../StorageApiWriteRecordsInconsistent.java | 50 +-
.../bigquery/StorageApiWriteUnshardedRecords.java | 303 ++++++++---
.../bigquery/StorageApiWritesShardedRecords.java | 344 ++++++++-----
.../beam/sdk/io/gcp/testing/BigqueryClient.java | 4 +-
.../sdk/io/gcp/testing/FakeDatasetService.java | 35 +-
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 21 +-
.../io/gcp/bigquery/BigQueryNestedRecordsIT.java | 5 +-
.../io/gcp/bigquery/BigQueryServicesImplTest.java | 36 +-
.../gcp/bigquery/StorageApiSinkFailedRowsIT.java | 266 ++++++++++
.../gcp/bigquery/TableRowToStorageApiProtoIT.java | 11 +-
.../SpannerChangeStreamErrorTest.java | 4 +-
sdks/java/io/hadoop-format/build.gradle | 40 --
.../apache/beam/sdk/io/jms/JmsCheckpointMark.java | 24 +-
.../java/org/apache/beam/sdk/io/jms/JmsIO.java | 111 +++-
.../java/org/apache/beam/sdk/io/jms/JmsIOTest.java | 112 ++++-
sdks/java/io/sparkreceiver/build.gradle | 1 -
.../ReadFromSparkReceiverWithOffsetDoFn.java | 52 +-
.../io/sparkreceiver/ArrayBufferDataReceiver.java | 85 ++++
.../io/sparkreceiver/ByteBufferDataReceiver.java | 84 ++++
.../sdk/io/sparkreceiver/IteratorDataReceiver.java | 87 ++++
.../sdk/io/sparkreceiver/SparkReceiverIOTest.java | 89 +++-
.../maven-archetypes/examples/generate-sources.sh | 10 +
.../src/main/resources/archetype-resources/pom.xml | 7 +
sdks/java/testing/load-tests/build.gradle | 15 +-
sdks/java/testing/nexmark/build.gradle | 47 +-
sdks/java/testing/tpcds/README.md | 4 +-
sdks/java/testing/tpcds/build.gradle | 2 +-
sdks/python/apache_beam/__init__.py | 2 +-
sdks/python/apache_beam/coders/coder_impl.pxd | 6 +-
.../anomaly_detection_pipeline}/__init__.py | 0
.../anomaly_detection_pipeline/config.py | 49 ++
.../anomaly_detection_pipeline/main.py | 144 ++++++
.../pipeline}/__init__.py | 0
.../anomaly_detection_pipeline/pipeline/options.py | 62 +++
.../pipeline/transformations.py | 194 +++++++
.../anomaly_detection_pipeline/setup.py | 49 ++
.../write_data_to_pubsub_pipeline}/__init__.py | 0
.../write_data_to_pubsub_pipeline/config.py} | 10 +-
.../write_data_to_pubsub_pipeline/main.py | 89 ++++
.../pipeline}/__init__.py | 0
.../pipeline/options.py | 62 +++
.../pipeline/utils.py | 67 +++
.../write_data_to_pubsub_pipeline/setup.py} | 28 +-
.../apache_beam/examples/wordcount_it_test.py | 41 +-
sdks/python/apache_beam/internal/gcp/auth.py | 6 +
sdks/python/apache_beam/io/gcp/bigquery.py | 5 +-
.../apache_beam/io/gcp/bigquery_read_it_test.py | 85 +++-
.../apache_beam/io/gcp/bigquery_schema_tools.py | 29 +-
.../io/gcp/bigquery_schema_tools_test.py | 21 +-
sdks/python/apache_beam/io/gcp/gcsio.py | 8 +
.../apache_beam/io/gcp/pubsub_io_perf_test.py | 8 +-
sdks/python/apache_beam/ml/gcp/visionml.py | 51 +-
sdks/python/apache_beam/ml/gcp/visionml_test.py | 9 +-
sdks/python/apache_beam/ml/gcp/visionml_test_it.py | 7 +-
.../python/apache_beam/options/pipeline_options.py | 18 +-
.../runners/dataflow/dataflow_runner.py | 30 +-
.../runners/dataflow/internal/apiclient.py | 2 +-
.../runners/dataflow/internal/apiclient_test.py | 2 +-
.../apache_beam/runners/dataflow/internal/names.py | 4 +-
.../apache_beam/runners/interactive/sql/utils.py | 3 +-
.../runners/interactive/sql/utils_test.py | 2 +-
.../runners/portability/expansion_service_test.py | 12 +-
.../portability/fn_api_runner/fn_runner_test.py | 3 +-
.../runners/portability/portable_runner_test.py | 3 +-
.../apache_beam/runners/portability/stager.py | 26 +-
.../apache_beam/runners/portability/stager_test.py | 43 ++
.../apache_beam/runners/worker/sdk_worker_main.py | 22 +-
.../runners/worker/sdk_worker_main_test.py | 35 +-
.../apache_beam/testing/load_tests/build.gradle | 1 +
.../apache_beam/transforms/batch_dofn_test.py | 3 +-
sdks/python/apache_beam/transforms/combiners.py | 17 +-
.../apache_beam/transforms/ptransform_test.py | 34 +-
.../typehints/native_type_compatibility.py | 6 +
sdks/python/apache_beam/typehints/opcodes.py | 6 +
.../typehints/pytorch_type_compatibility.py | 140 ++++++
.../typehints/pytorch_type_compatibility_test.py | 138 +++++
sdks/python/apache_beam/typehints/row_type.py | 3 +-
sdks/python/apache_beam/typehints/schemas.py | 17 +-
sdks/python/apache_beam/typehints/schemas_test.py | 4 +-
.../apache_beam/typehints/sharded_key_type.py | 8 +-
.../apache_beam/typehints/sharded_key_type_test.py | 8 +-
.../apache_beam/typehints/trivial_inference.py | 18 +-
.../typehints/trivial_inference_test.py | 8 +
.../python/apache_beam/typehints/typecheck_test.py | 6 +-
.../apache_beam/typehints/typed_pipeline_test.py | 20 +-
sdks/python/apache_beam/typehints/typehints.py | 66 +--
.../python/apache_beam/typehints/typehints_test.py | 111 ++--
sdks/python/apache_beam/version.py | 2 +-
sdks/python/build-requirements.txt | 4 +
.../container/base_image_requirements_manual.txt | 2 +-
sdks/python/container/build.gradle | 3 +
.../{py38 => py310}/base_image_requirements.txt | 51 +-
.../2 => sdks/python/container/py310}/build.gradle | 21 +-
.../container/py37/base_image_requirements.txt | 43 +-
.../container/py38/base_image_requirements.txt | 45 +-
.../container/py39/base_image_requirements.txt | 45 +-
sdks/python/container/run_validatescontainer.sh | 2 +-
sdks/python/scripts/run_integration_test.sh | 14 +
sdks/python/setup.py | 25 +-
sdks/python/test-suites/dataflow/common.gradle | 4 +
.../test-suites/dataflow/py310}/build.gradle | 19 +-
.../python/test-suites/direct/py310}/build.gradle | 19 +-
sdks/python/test-suites/gradle.properties | 32 +-
sdks/python/test-suites/portable/common.gradle | 7 +
.../test-suites/portable/py310}/build.gradle | 21 +-
.../python/test-suites/tox/py310}/build.gradle | 27 +-
sdks/python/tox.ini | 17 +-
settings.gradle.kts | 6 +
website/www/check-links.sh | 2 +-
website/www/site/config.toml | 2 +-
website/www/site/content/en/blog/beam-2.42.0.md | 127 +++++
website/www/site/content/en/case-studies/Amazon.md | 18 +
website/www/site/content/en/case-studies/ML6.md | 18 +
website/www/site/content/en/case-studies/Strivr.md | 17 +
.../www/site/content/en/case-studies/TrustPilot.md | 17 +
.../www/site/content/en/case-studies/Twitter.md | 17 +
.../www/site/content/en/case-studies/Wayfair.md | 17 +
.../www/site/content/en/case-studies/Wizeline.md | 17 +
.../en/documentation/ml/anomaly-detection.md | 238 +++++++++
.../content/en/documentation/ml/data-processing.md | 116 +++++
.../en/documentation/ml/multi-model-pipelines.md | 2 +-
.../site/content/en/documentation/ml/overview.md | 4 +
.../content/en/documentation/programming-guide.md | 47 +-
.../en/documentation/runtime/environments.md | 2 +
.../sdks/java-multi-language-pipelines.md | 79 ++-
.../documentation/sdks/python-machine-learning.md | 99 ++--
.../sdks/python-pipeline-dependencies.md | 3 +-
.../www/site/content/en/get-started/downloads.md | 10 +-
.../partials/section-menu/en/documentation.html | 2 +
.../site/static/images/logos/powered-by/Amazon.png | Bin 0 -> 38653 bytes
.../site/static/images/logos/powered-by/ML6.jpg | Bin 0 -> 6110 bytes
.../site/static/images/logos/powered-by/Strivr.png | Bin 0 -> 5686 bytes
.../static/images/logos/powered-by/Trustpilot.png | Bin 0 -> 25458 bytes
.../static/images/logos/powered-by/Twitter.png | Bin 0 -> 9561 bytes
.../static/images/logos/powered-by/Wayfair.png | Bin 0 -> 82679 bytes
.../static/images/logos/powered-by/Wizeline.png | Bin 0 -> 6981 bytes
276 files changed, 7969 insertions(+), 2190 deletions(-)
create mode 100644 .github/actions/common-rc-validation/action.yaml
create mode 100644 .github/workflows/assign_milestone.yml
copy sdks/python/apache_beam/version.py => .github/workflows/playground_examples_cd.yml (53%)
create mode 100644 .github/workflows/playground_examples_cd_reusable.yml
create mode 100644 .github/workflows/playground_examples_ci.yml
create mode 100644 .github/workflows/playground_examples_ci_reusable.yml
create mode 100644 .github/workflows/run_rc_validation.yml
create mode 100644 .github/workflows/tour_of_beam_examples_ci.yml
rename examples/{multi-language => java}/src/main/java/org/apache/beam/examples/multilanguage/PythonDataframeWordCount.java (99%)
create mode 100644 examples/java/src/main/java/org/apache/beam/examples/multilanguage/SklearnMnistClassification.java
create mode 100644 playground/frontend/lib/modules/examples/components/filter/filter.dart
copy playground/frontend/lib/modules/examples/components/filter/{category_bubble.dart => tag_bubble.dart} (82%)
rename playground/frontend/lib/modules/examples/components/filter/{category_bubble.dart => type_bubble.dart} (93%)
delete mode 100644 playground/frontend/lib/modules/examples/components/filter/type_filter.dart
create mode 100644 playground/infrastructure/test_checker.py
copy sdks/go/pkg/beam/core/{core.go => runtime/xlangx/expand_test.go} (57%)
create mode 100644 sdks/go/pkg/beam/transforms/xlang/dataframe/dataframe.go
create mode 100644 sdks/go/pkg/beam/transforms/xlang/python/external.go
create mode 100644 sdks/go/test/integration/transforms/xlang/dataframe/dataframe.go
create mode 100644 sdks/go/test/integration/transforms/xlang/dataframe/dataframe_test.go
rename sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/{ => batch}/EmployeeBatchSink.java (95%)
rename sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/{ => batch}/EmployeeBatchSource.java (94%)
rename sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/{ => batch}/EmployeeInputFormat.java (97%)
rename sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/{ => batch}/EmployeeInputFormatProvider.java (93%)
rename sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/{ => batch}/EmployeeOutputFormat.java (88%)
rename sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/{ => batch}/EmployeeOutputFormatProvider.java (93%)
create mode 100644 sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/streaming/EmployeeReceiver.java
create mode 100644 sdks/java/io/cdap/src/test/java/org/apache/beam/sdk/io/cdap/streaming/EmployeeStreamingSource.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSinkFailedRowsIT.java
create mode 100644 sdks/java/io/sparkreceiver/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ArrayBufferDataReceiver.java
create mode 100644 sdks/java/io/sparkreceiver/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ByteBufferDataReceiver.java
create mode 100644 sdks/java/io/sparkreceiver/src/test/java/org/apache/beam/sdk/io/sparkreceiver/IteratorDataReceiver.java
copy {.test-infra/jenkins/dependency_check => sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline}/__init__.py (100%)
create mode 100644 sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline/config.py
create mode 100644 sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline/main.py
copy {.test-infra/jenkins/dependency_check => sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline/pipeline}/__init__.py (100%)
create mode 100644 sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline/pipeline/options.py
create mode 100644 sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline/pipeline/transformations.py
create mode 100644 sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline/setup.py
copy {.test-infra/jenkins/dependency_check => sdks/python/apache_beam/examples/inference/anomaly_detection/write_data_to_pubsub_pipeline}/__init__.py (100%)
copy sdks/python/apache_beam/{version.py => examples/inference/anomaly_detection/write_data_to_pubsub_pipeline/config.py} (79%)
create mode 100644 sdks/python/apache_beam/examples/inference/anomaly_detection/write_data_to_pubsub_pipeline/main.py
copy {.test-infra/jenkins/dependency_check => sdks/python/apache_beam/examples/inference/anomaly_detection/write_data_to_pubsub_pipeline/pipeline}/__init__.py (100%)
create mode 100644 sdks/python/apache_beam/examples/inference/anomaly_detection/write_data_to_pubsub_pipeline/pipeline/options.py
create mode 100644 sdks/python/apache_beam/examples/inference/anomaly_detection/write_data_to_pubsub_pipeline/pipeline/utils.py
copy sdks/python/apache_beam/{version.py => examples/inference/anomaly_detection/write_data_to_pubsub_pipeline/setup.py} (50%)
create mode 100644 sdks/python/apache_beam/typehints/pytorch_type_compatibility.py
create mode 100644 sdks/python/apache_beam/typehints/pytorch_type_compatibility_test.py
copy sdks/python/container/{py38 => py310}/base_image_requirements.txt (83%)
copy {runners/spark/2 => sdks/python/container/py310}/build.gradle (60%)
copy {runners/spark/2 => sdks/python/test-suites/dataflow/py310}/build.gradle (60%)
copy {runners/spark/2 => sdks/python/test-suites/direct/py310}/build.gradle (60%)
copy {runners/spark/2 => sdks/python/test-suites/portable/py310}/build.gradle (60%)
copy {runners/spark/2 => sdks/python/test-suites/tox/py310}/build.gradle (60%)
create mode 100644 website/www/site/content/en/blog/beam-2.42.0.md
create mode 100644 website/www/site/content/en/case-studies/Amazon.md
create mode 100644 website/www/site/content/en/case-studies/ML6.md
create mode 100644 website/www/site/content/en/case-studies/Strivr.md
create mode 100644 website/www/site/content/en/case-studies/TrustPilot.md
create mode 100644 website/www/site/content/en/case-studies/Twitter.md
create mode 100644 website/www/site/content/en/case-studies/Wayfair.md
create mode 100644 website/www/site/content/en/case-studies/Wizeline.md
create mode 100644 website/www/site/content/en/documentation/ml/anomaly-detection.md
create mode 100755 website/www/site/content/en/documentation/ml/data-processing.md
create mode 100644 website/www/site/static/images/logos/powered-by/Amazon.png
create mode 100644 website/www/site/static/images/logos/powered-by/ML6.jpg
create mode 100644 website/www/site/static/images/logos/powered-by/Strivr.png
create mode 100644 website/www/site/static/images/logos/powered-by/Trustpilot.png
create mode 100644 website/www/site/static/images/logos/powered-by/Twitter.png
create mode 100644 website/www/site/static/images/logos/powered-by/Wayfair.png
create mode 100644 website/www/site/static/images/logos/powered-by/Wizeline.png