You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2023/10/25 05:10:42 UTC
[beam] branch nightly-refs/heads/master updated (3f05945888e -> 8b31859ff8a)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 3f05945888e [YAML] Schema-producing Create. (#29030)
add 1a27d2cee0c flush buffer during drain operation for requiresStableInput operator (#28554)
add bf5ded44e6e Merge pull request #29102: [flink] Flush buffer during drain operation for requiresStableInput operator
add 9e21f3d0720 adding metrics report (#29010)
add bc06581e110 Enable run test on Java 21 (#28969)
add b332eeb2901 Add MetricsContainer support to the Flink sources.
add 3f2c5e76c07 Favor composition in FlinkMetricsContainer implementations
add 36ba537c6c2 Fix Guava import
add 8a2ed4d7f56 Add MetricsContainer support to the Flink sources. #28609
add 0d95b4b400e Increment Go build version to 1.21.3 (#29119)
add 9b2ca42910c Fix flatten docs (#29120)
add be2cd9dcdef change report generator to be aware of GH and passtrough vars to tox (#29121)
add 02b84b8487a Follow up Java21 examples (#29117)
add ad11e0a664a add auth to gcp step (#29123)
add 9d142879fb2 remove subdir from test config paths (#29125)
add 0478cd2827c move performance singlestoreio config to the right place (#29126)
add 0f769814326 Teardown Load test and IO Performance tests on Jenkins (#29092)
add 8a827e8c018 auth fix (#29128)
add 48722f1bfb4 [PYTHON] Add new `--auto_unique_labels` option to StandardOptions (#28984)
add 8b31859ff8a [Python]Remove get_artifacts in MLTranform since artifacts are stored in artifact location (#29016)
No new revisions were added by this update.
Summary of changes:
.github/workflows/README.md | 3 +-
...m_LoadTests_Python_CoGBK_Dataflow_Streaming.yml | 8 +-
.../beam_LoadTests_Python_CoGBK_Flink_Batch.yml | 6 +-
...a_InfluxDbIO_IT.yml => beam_Metrics_Report.yml} | 67 ++--
.../workflows/beam_PerformanceTests_Kafka_IO.yml | 6 +
.../beam_PerformanceTests_SingleStoreIO.yml | 2 +-
.../beam_PerformanceTests_xlang_KafkaIO_Python.yml | 6 +
...m_PostCommit_Java_Examples_Dataflow_V2_Java.yml | 7 +-
...=> beam_PostCommit_Java_Jpms_Direct_Java21.yml} | 67 ++--
.../config_PerformanceTests_SingleStoreIO.txt | 0
.../job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy | 246 -------------
.../job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy | 246 -------------
...ts_CoGBK_Java_spark_structured_streaming.groovy | 172 ---------
.../jenkins/job_LoadTests_Combine_Go.groovy | 127 -------
.../jenkins/job_LoadTests_Combine_Java.groovy | 162 ---------
.../job_LoadTests_Combine_Java_Smoke.groovy | 85 -----
..._Combine_Java_spark_structured_streaming.groovy | 123 -------
.../job_LoadTests_FnApiRunner_Python.groovy | 74 ----
.../job_LoadTests_GBK_Dataflow_V2_Java11.groovy | 311 -----------------
.../job_LoadTests_GBK_Dataflow_V2_Java17.groovy | 311 -----------------
.../jenkins/job_LoadTests_GBK_Flink_Python.groovy | 172 ---------
.test-infra/jenkins/job_LoadTests_GBK_Go.groovy | 227 ------------
.test-infra/jenkins/job_LoadTests_GBK_Java.groovy | 261 --------------
.../jenkins/job_LoadTests_GBK_Java_Smoke.groovy | 85 -----
...ests_GBK_Java_spark_structured_streaming.groovy | 210 -----------
.../jenkins/job_LoadTests_GBK_Python.groovy | 209 -----------
.../job_LoadTests_GBK_Python_reiterate.groovy | 137 --------
.../job_LoadTests_ParDo_Dataflow_V2_Java11.groovy | 217 ------------
.../job_LoadTests_ParDo_Dataflow_V2_Java17.groovy | 217 ------------
.../jenkins/job_LoadTests_ParDo_Flink_Go.groovy | 152 --------
.test-infra/jenkins/job_LoadTests_ParDo_Go.groovy | 155 ---------
.../jenkins/job_LoadTests_ParDo_Java.groovy | 186 ----------
...ts_ParDo_Java_spark_structured_streaming.groovy | 148 --------
.../jenkins/job_LoadTests_ParDo_Python.groovy | 186 ----------
.../jenkins/job_LoadTests_Python_Smoke.groovy | 71 ----
.../jenkins/job_LoadTests_SideInput_Go.groovy | 100 ------
.../jenkins/job_LoadTests_SideInput_Python.groovy | 198 -----------
.../job_LoadTests_coGBK_Flink_Python.groovy | 156 ---------
.test-infra/jenkins/job_LoadTests_coGBK_Go.groovy | 179 ----------
.../job_PerformanceTests_BigQueryIO_Java.groovy | 145 --------
.../job_PerformanceTests_BigQueryIO_Python.groovy | 110 ------
.../jenkins/job_PerformanceTests_CdapIO.groovy | 72 ----
.../job_PerformanceTests_FileBasedIO_IT.groovy | 384 ---------------------
.../job_PerformanceTests_FileBasedIO_Python.groovy | 82 -----
.../job_PerformanceTests_HadoopFormat.groovy | 73 ----
.../jenkins/job_PerformanceTests_JDBC.groovy | 73 ----
.../job_PerformanceTests_MongoDBIO_IT.groovy | 71 ----
.../job_PerformanceTests_PubsubIO_Python.groovy | 72 ----
.../jenkins/job_PerformanceTests_Python.groovy | 97 ------
.../jenkins/job_PerformanceTests_SQLIO_Java.groovy | 67 ----
.../job_PerformanceTests_SingleStoreIO.groovy | 86 -----
.../job_PerformanceTests_SpannerIO_Python.groovy | 112 ------
.../job_PerformanceTests_SparkReceiverIO_IT.groovy | 84 -----
.../jenkins/metrics_report/report_generator.py | 9 +-
.test-infra/jenkins/metrics_report/tox.ini | 2 +-
CHANGES.md | 4 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 192 +++++------
examples/java/build.gradle | 4 +
examples/kotlin/build.gradle | 9 +-
gradle/wrapper/gradle-wrapper.properties | 2 +-
gradlew | 14 +-
.../python/Core Transforms/Flatten/Flatten/task.py | 30 +-
.../streaming/io/source/SourceTestCompat.java | 13 +
.../streaming/io/source/SourceTestCompat.java | 13 +
.../beam/runners/flink/FlinkPipelineOptions.java | 8 +
.../beam/runners/flink/FlinkRunnerResult.java | 4 +-
.../flink/metrics/FlinkMetricContainer.java | 160 +--------
...ontainer.java => FlinkMetricContainerBase.java} | 74 ++--
.../FlinkMetricContainerWithoutAccumulator.java} | 15 +-
.../flink/metrics/ReaderInvocationUtil.java | 4 +-
.../wrappers/streaming/DoFnOperator.java | 14 +
.../wrappers/streaming/io/source/FlinkSource.java | 12 +-
.../streaming/io/source/FlinkSourceReaderBase.java | 13 +-
.../io/source/bounded/FlinkBoundedSource.java | 8 +-
.../source/bounded/FlinkBoundedSourceReader.java | 10 +-
.../io/source/unbounded/FlinkUnboundedSource.java | 13 +-
.../unbounded/FlinkUnboundedSourceReader.java | 6 +-
.../flink/metrics/FlinkMetricContainerTest.java | 2 +-
.../wrappers/streaming/DoFnOperatorTest.java | 92 +++++
.../wrappers/streaming/io/TestCountingSource.java | 7 +
.../io/source/FlinkSourceReaderTestBase.java | 28 ++
.../bounded/FlinkBoundedSourceReaderTest.java | 5 +-
.../unbounded/FlinkUnboundedSourceReaderTest.java | 5 +-
sdks/java/container/agent/build.gradle | 9 +-
sdks/java/container/java21/build.gradle | 2 -
sdks/java/testing/jpms-tests/build.gradle | 4 +-
.../transforms/elementwise/mltransform_test.py | 8 +-
sdks/python/apache_beam/ml/transforms/base.py | 13 -
.../apache_beam/ml/transforms/handlers_test.py | 18 -
sdks/python/apache_beam/ml/transforms/tft.py | 82 +----
sdks/python/apache_beam/ml/transforms/tft_test.py | 112 ++----
.../python/apache_beam/options/pipeline_options.py | 8 +
sdks/python/apache_beam/pipeline.py | 33 +-
sdks/python/apache_beam/pipeline_test.py | 27 ++
sdks/python/tox.ini | 9 +-
.../content/en/documentation/ml/preprocess-data.md | 5 +-
.../shortcodes/flink_java_pipeline_options.html | 5 +
.../shortcodes/flink_python_pipeline_options.html | 5 +
98 files changed, 620 insertions(+), 7315 deletions(-)
copy .github/workflows/{beam_PostCommit_Java_InfluxDbIO_IT.yml => beam_Metrics_Report.yml} (59%)
copy .github/workflows/{beam_PreCommit_Java_Google-ads_IO_Direct.yml => beam_PostCommit_Java_Jpms_Direct_Java21.yml} (67%)
rename .github/workflows/{performance-tests-job-configs => performance-tests-pipeline-options}/config_PerformanceTests_SingleStoreIO.txt (100%)
delete mode 100644 .test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_CoGBK_Java_spark_structured_streaming.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_Combine_Go.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_Combine_Java.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_Combine_Java_Smoke.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_Combine_Java_spark_structured_streaming.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_FnApiRunner_Python.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java11.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java17.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_GBK_Flink_Python.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_GBK_Go.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_GBK_Java.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_GBK_Java_Smoke.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_GBK_Java_spark_structured_streaming.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_GBK_Python.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_GBK_Python_reiterate.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java11.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java17.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_ParDo_Flink_Go.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_ParDo_Go.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_ParDo_Java.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_ParDo_Java_spark_structured_streaming.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_ParDo_Python.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_Python_Smoke.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_SideInput_Go.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_SideInput_Python.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_coGBK_Flink_Python.groovy
delete mode 100644 .test-infra/jenkins/job_LoadTests_coGBK_Go.groovy
delete mode 100644 .test-infra/jenkins/job_PerformanceTests_BigQueryIO_Java.groovy
delete mode 100644 .test-infra/jenkins/job_PerformanceTests_BigQueryIO_Python.groovy
delete mode 100644 .test-infra/jenkins/job_PerformanceTests_CdapIO.groovy
delete mode 100644 .test-infra/jenkins/job_PerformanceTests_FileBasedIO_IT.groovy
delete mode 100644 .test-infra/jenkins/job_PerformanceTests_FileBasedIO_Python.groovy
delete mode 100644 .test-infra/jenkins/job_PerformanceTests_HadoopFormat.groovy
delete mode 100644 .test-infra/jenkins/job_PerformanceTests_JDBC.groovy
delete mode 100644 .test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy
delete mode 100644 .test-infra/jenkins/job_PerformanceTests_PubsubIO_Python.groovy
delete mode 100644 .test-infra/jenkins/job_PerformanceTests_Python.groovy
delete mode 100644 .test-infra/jenkins/job_PerformanceTests_SQLIO_Java.groovy
delete mode 100644 .test-infra/jenkins/job_PerformanceTests_SingleStoreIO.groovy
delete mode 100644 .test-infra/jenkins/job_PerformanceTests_SpannerIO_Python.groovy
delete mode 100644 .test-infra/jenkins/job_PerformanceTests_SparkReceiverIO_IT.groovy
copy runners/flink/src/main/java/org/apache/beam/runners/flink/metrics/{FlinkMetricContainer.java => FlinkMetricContainerBase.java} (70%)
copy runners/flink/{1.12/src/test/java/org/apache/beam/runners/flink/metrics/MetricGroupWrapper.java => src/main/java/org/apache/beam/runners/flink/metrics/FlinkMetricContainerWithoutAccumulator.java} (62%)