You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2025/06/26 20:20:04 UTC
(beam) branch users/damccorm/prismByDefault updated (2410ca63003 -> 051a4ed37b0)
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a change to branch users/damccorm/prismByDefault
in repository https://gitbox.apache.org/repos/asf/beam.git
from 2410ca63003 Bump workflow timeout (#35420)
add 46bb8a9892c linting/fixes
add 2d5cd0723c0 Merge branch 'users/damccorm/prismByDefault' of https://github.com/apache/beam into users/damccorm/prismByDefault
add 5572ad8b04e Allow STRING format in Kafka config
add 2b4056bcbb3 Merge pull request #35214: Allow STRING format in Kafka config
add 33ea9388fb9 Use now + 2mins as the end timestamp for change stream read API if the (#34967)
add 5532af8438f Update JupyterLab Sidepanel yarn.lock file (#35195)
add b09e5ef4bcd Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#35366)
add b74c4960216 estimate request size in BQ streaming inserts when with_auto_sharding is true (#35212)
add 0d6c94a8727 Fix PubsubIOLT (#35372)
add 8c832e93423 Fix the issue of sliding window firing early. (#35385)
add 2feb407ea0e Fix suppressed Spotbugs issues
add 4c1c2daa17d Fix suppressed Spotbugs issues (#35394)
add 01411e71197 Bump google.golang.org/api from 0.237.0 to 0.238.0 in /sdks (#35368)
add f2d0dc8aecc Improve LogElements to show pane_info and timestamps in seconds. (#35387)
add dc213a998c3 Add change stream new record models and tests. (#35258)
add 70a066bfc12 Use splitAsStream for wordcount (#35358)
add d166a1ab6e8 Improve exception raised from cache (#35403)
add 4bfa0c90fb8 Fix(yaml): Handle missing optional fields in JSON parsing (#35288)
add 69e996f733d improved the error message when encoding_positions_set has errors (#35410)
add 3b5018ad4d0 Reduce issues induced by ordering (#35405)
add fbc00451c3a Fix buildx always enabled (#35361)
add 6d0e00ea617 added defaultQueryTimeout for jdbc io (#35376)
add b97ee3110d0 [yaml]: Phase2 High Usage Yaml examples (#35279)
add cb5eef5b6f1 ✨[Feature]: Add pip-based install support for JupyterLab side panel extension (#35399)
add 0595584d986 Skip sliding window tests on samza and spark runner test suites (#35430)
add ac350626fa4 Pin torch dependencies to tested versions (#35431)
add 6ac5009dbfd Fix post commit xlang jdbcio test failure (#35428)
add 944aaa0e1f9 Disable dependency logging in local envs (#35404)
add ffd89458a85 add filter to watermark query (#35042)
add 4a7a9a39d31 Fix data race ExecutionStateSampler.java:491 in ExecutionStateImpl.reset() (#35135)
add 4e91f2c60e5 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#35443)
add 2b9c7b60060 Fix internal flume test failure due to a newly added test. (#35440)
add 4070b07c5ad Bump org.ajoberstar.grgit:grgit-gradle from 4.1.1 to 5.3.2 (#35324)
add 7b63aecde89 Removed < 1.66 for grpcio (#34196)
add 273a694a510 Move to Java11 for GHA workflow still uses Java 8 (#35446)
add 89240df5d22 Revert "Bump org.ajoberstar.grgit:grgit-gradle from 4.1.1 to 5.3.2 (#35324)" (#35450)
add 9331d411806 Bring the Python Tips document into the main repository (#35447)
add 048405d797f Add display_data to AppliedPTransforms (#35380)
add 051a4ed37b0 Merge in master
No new revisions were added by this update.
Summary of changes:
...am_PostCommit_Python_ValidatesRunner_Samza.json | 3 +-
...am_PostCommit_Python_ValidatesRunner_Spark.json | 3 +-
...LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml | 2 +-
...Tests_Java_GBK_Dataflow_V2_Streaming_Java17.yml | 2 +-
.../workflows/beam_PostCommit_Java_DataflowV1.yml | 4 +-
.../beam_PostCommit_Java_Jpms_Dataflow_Java17.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Direct_Java17.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Direct_Java21.yml | 2 +-
...stCommit_Python_ValidatesContainer_Dataflow.yml | 4 +-
..._Python_ValidatesContainer_Dataflow_With_RC.yml | 4 +-
.../workflows/beam_PreCommit_Flink_Container.yml | 2 +-
...eam_PreCommit_Java_Examples_Dataflow_Java21.yml | 2 +-
.github/workflows/beam_PreCommit_SQL_Java17.yml | 2 +-
.github/workflows/build_release_candidate.yml | 4 +-
CHANGES.md | 2 +
.../org/apache/beam/gradle/BeamDockerPlugin.groovy | 6 +-
.../apache/beam/gradle/BeamJenkinsPlugin.groovy | 56 -
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 8 +-
.../images/unreleased_changes_screen.png | Bin 0 -> 113414 bytes
contributor-docs/python-tips.md | 515 +
.../java/org/apache/beam/examples/WordCount.java | 19 +-
.../datatokenization/utils/SchemasUtils.java | 8 +-
.../subprocess/SubProcessPipelineOptions.java | 2 +-
.../configuration/SubProcessConfiguration.java | 12 +-
.../examples/kotlin/cookbook/TriggerExample.kt | 3 +-
.../org/apache/beam/it/gcp/pubsub/PubsubIOLT.java | 9 +-
sdks/go.mod | 4 +-
sdks/go.sum | 8 +-
.../prism/internal/engine/elementmanager.go | 29 +-
.../fn/harness/control/ExecutionStateSampler.java | 11 +-
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 4 +-
.../action/QueryChangeStreamAction.java | 23 +-
.../changestreams/cache/AsyncWatermarkCache.java | 21 +-
.../changestreams/cache/NoOpWatermarkCache.java | 2 +-
.../changestreams/dao/PartitionMetadataDao.java | 31 +-
...rtitionsRecord.java => PartitionEndRecord.java} | 80 +-
...itionsRecord.java => PartitionEventRecord.java} | 79 +-
...itionsRecord.java => PartitionStartRecord.java} | 61 +-
.../SpannerChangeStreamErrorTest.java | 12 +-
.../changestreams/it/SpannerChangeStreamIT.java | 3 +
...StreamOrderedByTimestampAndTransactionIdIT.java | 3 +
...nnerChangeStreamOrderedWithinKeyGloballyIT.java | 3 +
.../it/SpannerChangeStreamOrderedWithinKeyIT.java | 3 +
.../it/SpannerChangeStreamPostgresIT.java | 3 +
...SpannerChangeStreamTransactionBoundariesIT.java | 3 +
.../it/SpannerChangeStreamsSchemaTransformIT.java | 3 +
...RecordTest.java => PartitionEndRecordTest.java} | 18 +-
...cordTest.java => PartitionEventRecordTest.java} | 18 +-
...cordTest.java => PartitionStartRecordTest.java} | 18 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 23 +
.../beam/sdk/io/kafka/KafkaExactlyOnceSink.java | 5 +-
.../org/apache/beam/sdk/io/kafka/KafkaIOUtils.java | 7 +-
.../KafkaReadSchemaTransformConfiguration.java | 3 +
.../KafkaReadSchemaTransformProviderTest.java | 25 +-
.../beam/sdk/io/synthetic/SyntheticStep.java | 6 +-
sdks/python/README.md | 2 +-
sdks/python/apache_beam/coders/coder_impl.py | 4 +-
.../io/external/xlang_jdbcio_it_test.py | 10 +
sdks/python/apache_beam/io/gcp/bigquery.py | 73 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 141 +
sdks/python/apache_beam/io/parquetio_test.py | 1 -
.../python/apache_beam/options/pipeline_options.py | 9 +
sdks/python/apache_beam/pipeline.py | 10 +-
.../apache-beam-jupyterlab-sidepanel/.gitignore | 6 +
.../apache-beam-jupyterlab-sidepanel/.yarnrc.yml | 4 +-
.../apache-beam-jupyterlab-sidepanel/README.md | 100 +-
.../apache_beam_jupyterlab_sidepanel/__init__.py | 7 +-
.../apache_beam_jupyterlab_sidepanel/_version.py | 6 +-
.../apache-beam-jupyterlab-sidepanel/package.json | 3 +-
.../pyproject.toml | 67 +
.../apache-beam-jupyterlab-sidepanel/yarn.lock | 17809 +++++++++++--------
.../apache_beam/runners/pipeline_utils_test.py | 5 +-
.../portability/fn_api_runner/fn_runner_test.py | 25 +
.../runners/portability/samza_runner_test.py | 3 +
.../runners/portability/sdk_container_builder.py | 3 +-
.../runners/portability/spark_runner_test.py | 3 +
sdks/python/apache_beam/transforms/display.py | 6 +-
sdks/python/apache_beam/transforms/display_test.py | 8 +
sdks/python/apache_beam/transforms/environments.py | 8 +-
.../apache_beam/transforms/ptransform_test.py | 87 +-
sdks/python/apache_beam/transforms/util.py | 42 +-
sdks/python/apache_beam/transforms/util_test.py | 52 +-
.../yaml/examples/testing/examples_test.py | 119 +-
.../yaml/examples/testing/input_data.py | 71 +-
.../transforms/blueprint/jdbc_to_bigquery.yaml | 57 +
.../blueprint/pubsub_subscription_to_bigquery.yaml | 61 +
.../blueprint/pubsub_topic_to_bigquery.yaml | 61 +
.../transforms/blueprint/spanner_to_avro.yaml | 45 +
sdks/python/apache_beam/yaml/json_utils.py | 29 +-
sdks/python/apache_beam/yaml/json_utils_test.py | 157 +
sdks/python/setup.py | 9 +-
sdks/python/tox.ini | 6 +
92 files changed, 12035 insertions(+), 8261 deletions(-)
delete mode 100644 buildSrc/src/main/groovy/org/apache/beam/gradle/BeamJenkinsPlugin.groovy
create mode 100644 contributor-docs/images/unreleased_changes_screen.png
create mode 100644 contributor-docs/python-tips.md
copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/{ChildPartitionsRecord.java => PartitionEndRecord.java} (51%)
copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/{ChildPartitionsRecord.java => PartitionEventRecord.java} (52%)
copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/{ChildPartitionsRecord.java => PartitionStartRecord.java} (62%)
copy sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/{HeartbeatRecordTest.java => PartitionEndRecordTest.java} (73%)
copy sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/{HeartbeatRecordTest.java => PartitionEventRecordTest.java} (72%)
copy sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/{ChildPartitionsRecordTest.java => PartitionStartRecordTest.java} (72%)
create mode 100644 sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/.gitignore
copy .test-infra/metrics/sync/jenkins/requirements.txt => sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/.yarnrc.yml (96%)
copy .test-infra/kubernetes/hadoop/LargeITCluster/teardown.sh => sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/apache_beam_jupyterlab_sidepanel/__init__.py (84%)
mode change 100755 => 100644
copy .test-infra/metrics/sync/jenkins/requirements.txt => sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/apache_beam_jupyterlab_sidepanel/_version.py (95%)
create mode 100644 sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/pyproject.toml
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/blueprint/jdbc_to_bigquery.yaml
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/blueprint/pubsub_subscription_to_bigquery.yaml
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/blueprint/pubsub_topic_to_bigquery.yaml
create mode 100644 sdks/python/apache_beam/yaml/examples/transforms/blueprint/spanner_to_avro.yaml
create mode 100644 sdks/python/apache_beam/yaml/json_utils_test.py