You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2022/11/19 04:41:18 UTC
[beam] branch nightly-refs/heads/master updated (245fea90401 -> 36b79851359)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 245fea90401 Bump loader-utils from 1.4.1 to 1.4.2 in /sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel (#24191)
add 959719d01c6 Temporary update Python RC validation job
add c2feb09ea49 updates
add b7e860a7621 updates
add 79ba69483a8 Merge pull request #24218: Update Python wheel format for RC validation
add a0146371069 Uses _all to follow alias/datastreams when estimating index size
add ce8d618c77d Adds test for following aliases when estimating index size
add bff4919ff00 Merge pull request #24186: Uses _all to follow alias/datastreams when estimating index size
add 755b9c9d3e8 Bump github.com/aws/aws-sdk-go-v2/config from 1.18.0 to 1.18.1 in /sdks (#24222)
add 8cd94c5c439 Add enableGzipHttpCompression option to SplunkIO (#24197)
add fb5418b1ef5 Unify test parameters for certain IOs based on test row and grafana fixes
add 33bcf89ac2f Merge pull request #24211: Unify test parameters for certain IOs based on test row and Grafana fixes
add 2dc6e2f8ca3 [Metrics] Add new performance dashboard for Java JMH benchmarks (closes #22238)
add 2a8842b7dad Merge pull request #24032: [Metrics] Add new performance dashboard for Java JMH benchmarks
add 62c40292885 [Playground] Examples CI restore (#24155)
add 35596b4ab55 Bump github.com/aws/aws-sdk-go-v2/service/s3 in /sdks (#24220)
add 42688356bd2 Force discarding mode in with_fanout without rewindowing. (#23828)
add 60e91a9000e Fix checkArgument format string in BigQueryQueryHelper
add 44f7f96e966 Merge pull request #24233: Fix checkArgument format string in BigQueryQueryHelper
add c5e73352926 Clarify that SDF authors need to make the restriction sizing method thread safe if they override the default. (#24068)
add 16ee0fee9a3 Remove google-cloud-platform-core dependency from harness (#24235)
add 000f77fdd03 Issue#24161 Updated docstring for Clusters class
add c2442b4cf04 Removed trailing whitespaces.
add 2a7584fe4d8 Addresses Issue#24161 Updated docstrings for Clusters class
add e41ff28c16f Document our benchmarks (#24216)
add 886577fb35e Fix checkArgument calls in BQ dynamic destinations
add 5761bb0f993 Merge pull request #24234: Fix checkArgument calls in BQ dynamic destinations
add 239fe6a08a4 Website updates for Beam 2.43.0 release (#24044)
add 4c4e8474bbe Add @RunWith annotation to BQ test class
add 1dd2ccd0437 Merge pull request: Add @RunWith annotation to BQ test class
add eb552cd1335 Fix checkArgument format string in ByteKeyRange
add c22446adf39 Merge pull request #24133: Fix checkArgument format string in ByteKeyRange
add 66df9472b99 Fix release date
add 27b4a549d8a Few more fixes to the Website
add cf56af2799f Merge pull request #24246: Several fixes to the Website regarding the Beam 2.43.0 release
add a457b0669fa Move dashboard links to dedicated section
add 217e697d6ca [Spark dataset runner] Cache datasets if used multiple times (#24009)
add 4cb3a6b0afc Remove section from troubleshooting about fixed dictionary issue (#23152)
add 60caac6dc55 Fix flink XVR tests (#24228)
add 90fffc0fbd8 Adding the list of example notebooks to the ML readme file. (#24255)
add 902b4ecfe9a Updating timezone for Beam 2.43.0 release (#24258)
add d51c78b93df Only skip checkerframework if explicitly requested
add 7d1b2af31ea Merge pull request #24253: Only skip checkerframework if explicitly requested
add 02656348380 SingleStoreIO (#23535)
add edfa5cacfc2 Issue#21430 Updated dataframe io to avoid pruning
add c54082f291a Addresses #21430 Updated dataframe io to avoid pruning
add 4a06f1db43f Add @RunWith annotation to pubsublite test
add 10d63da147b Merge pull request #24240: Add @RunWith annotation to pubsublite test
add 6f3f2794fa1 Remove overly broad CanIgnoreReturnValue
add 0310365db5e Merge pull request #24236: Remove overly broad CanIgnoreReturnValue
add 0b55b766915 Update Java Multi-lang quickstart after the Beam 2.43.0 release
add 36b79851359 Merge pull request #24270: Update Java Multi-lang quickstart after the Beam 2.43.0 release
No new revisions were added by this update.
Summary of changes:
.github/workflows/playground_examples_ci.yml | 11 +-
.../workflows/playground_examples_ci_reusable.yml | 31 +-
.../workflows/tour_of_beam_backend_integration.yml | 10 +-
.../jenkins/job_PerformanceTests_CdapIO.groovy | 4 +-
.../job_PerformanceTests_HadoopFormat.groovy | 4 +-
.../jenkins/job_PerformanceTests_JDBC.groovy | 2 +-
.../job_PerformanceTests_SingleStoreIO.groovy | 86 +
.../job_PerformanceTests_SparkReceiverIO_IT.groovy | 4 +-
.../kubernetes/singlestore/sdb-cluster-crd.yaml | 78 +
.../kubernetes/singlestore/sdb-cluster.yaml | 54 +
.../kubernetes/singlestore/sdb-operator.yaml | 58 +
.test-infra/kubernetes/singlestore/sdb-rbac.yaml | 102 ++
.../singlestore/wait-for-pod-with-label.sh | 48 +
.test-infra/kubernetes/singlestore/wait-for.sh | 48 +
.../Java_IO_IT_Tests_Dataflow.json | 137 +-
.../perftests_metrics/Java_JMH_benchmarks.json | 1640 ++++++++++++++++++++
.../Python_IO_IT_Tests_Dataflow.json | 6 +-
.../Python_ML_RunInference_Benchmark_Tests.json | 7 +-
CHANGES.md | 15 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 12 +-
examples/notebooks/beam-ml/README.md | 24 +
playground/backend/containers/go/Dockerfile | 2 +-
playground/backend/containers/python/Dockerfile | 3 +-
playground/backend/containers/python/build.gradle | 2 +
.../python_release_automation_utils.sh | 8 +-
runners/flink/job-server/flink_job_server.gradle | 10 +-
.../SparkStructuredStreamingRunner.java | 34 +-
.../aggregators/AggregatorsAccumulator.java | 6 +-
.../metrics/MetricsAccumulator.java | 6 +-
.../translation/EvaluationContext.java | 120 ++
.../translation/PipelineTranslator.java | 380 +++--
.../translation/TransformTranslator.java | 118 +-
.../translation/TranslationContext.java | 141 --
.../CreatePCollectionViewTranslatorBatch.java | 46 -
.../translation/batch/ParDoTranslatorBatch.java | 15 +-
.../translation/batch/PipelineTranslatorBatch.java | 25 +-
.../translation/helpers/EncoderProvider.java | 58 +
sdks/go.mod | 16 +-
sdks/go.sum | 16 +-
sdks/go/pkg/beam/pardo.go | 4 +-
.../org/apache/beam/sdk/io/range/ByteKeyRange.java | 2 +-
.../java/org/apache/beam/sdk/transforms/DoFn.java | 3 +
sdks/java/harness/build.gradle | 1 -
.../java/org/apache/beam/fn/harness/FnHarness.java | 8 +-
.../fn/harness/control/ExecutionStateSampler.java | 7 +-
.../fn/harness/logging/BeamFnLoggingClient.java | 4 +-
.../stream/HarnessStreamObserverFactories.java | 6 +-
.../org/apache/beam/fn/harness/FnHarnessTest.java | 6 +-
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 8 +
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 8 +
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 8 +
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 8 +
.../io/elasticsearch/ElasticsearchIOTestUtils.java | 35 +-
.../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 17 +-
.../sdk/io/gcp/bigquery/BigQueryQueryHelper.java | 2 +-
.../sdk/io/gcp/bigquery/CreateTableHelpers.java | 7 +-
.../sdk/io/gcp/pubsublite/PublisherOptions.java | 2 +-
.../sdk/io/gcp/pubsublite/SubscriberOptions.java | 2 +-
.../gcp/pubsublite/UuidDeduplicationOptions.java | 2 +-
.../BigQuerySchemaTransformWriteProviderTest.java | 3 +
.../internal/BlockingCommmitterImplTest.java | 3 +
.../internal/CheckpointMarkImplTest.java | 3 +
.../internal/UnboundedReaderImplTest.java | 3 +
sdks/java/io/singlestore/build.gradle | 43 +
.../beam/sdk/io/singlestore/SingleStoreIO.java | 955 ++++++++++++
.../beam/sdk/io/singlestore/SingleStoreUtil.java | 92 ++
.../beam/sdk/io/singlestore/package-info.java} | 31 +-
.../singlestore/DataSourceConfigurationTest.java | 45 +
.../apache/beam/sdk/io/singlestore/ReadTest.java | 216 +++
.../sdk/io/singlestore/ReadWithPartitionsTest.java | 228 +++
.../beam/sdk/io/singlestore/SingleStoreIOIT.java | 268 ++++
.../SingleStoreIOTestPipelineOptions.java | 50 +
.../sdk/io/singlestore/SingleStoreUtilTest.java | 153 ++
.../apache/beam/sdk/io/singlestore/TestHelper.java | 70 +
.../apache/beam/sdk/io/singlestore/WriteTest.java | 265 ++++
.../beam/sdk/io/splunk/HttpEventPublisher.java | 22 +
.../beam/sdk/io/splunk/SplunkEventWriter.java | 140 +-
.../org/apache/beam/sdk/io/splunk/SplunkIO.java | 71 +-
.../beam/sdk/io/splunk/HttpEventPublisherTest.java | 7 +-
sdks/python/apache_beam/dataframe/io.py | 2 +-
.../apache_beam/examples/inference/README.md | 2 +
.../runners/interactive/interactive_beam.py | 13 +-
.../testing/benchmarks/inference/README.md | 53 +
.../apache_beam/transforms/combiners_test.py | 61 +
sdks/python/apache_beam/transforms/core.py | 17 +-
sdks/python/scripts/run_job_server.sh | 7 +-
settings.gradle.kts | 1 +
website/www/site/config.toml | 2 +-
website/www/site/content/en/blog/beam-2.43.0.md | 151 ++
.../en/documentation/ml/runinference-metrics.md | 2 +-
.../content/en/documentation/programming-guide.md | 4 +-
.../sdks/java-multi-language-pipelines.md | 84 +-
.../documentation/sdks/python-machine-learning.md | 9 +-
.../www/site/content/en/get-started/downloads.md | 16 +-
94 files changed, 5989 insertions(+), 630 deletions(-)
create mode 100644 .test-infra/jenkins/job_PerformanceTests_SingleStoreIO.groovy
create mode 100644 .test-infra/kubernetes/singlestore/sdb-cluster-crd.yaml
create mode 100644 .test-infra/kubernetes/singlestore/sdb-cluster.yaml
create mode 100644 .test-infra/kubernetes/singlestore/sdb-operator.yaml
create mode 100644 .test-infra/kubernetes/singlestore/sdb-rbac.yaml
create mode 100755 .test-infra/kubernetes/singlestore/wait-for-pod-with-label.sh
create mode 100755 .test-infra/kubernetes/singlestore/wait-for.sh
create mode 100644 .test-infra/metrics/grafana/dashboards/perftests_metrics/Java_JMH_benchmarks.json
create mode 100644 runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/EvaluationContext.java
delete mode 100644 runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
delete mode 100644 runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CreatePCollectionViewTranslatorBatch.java
create mode 100644 runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderProvider.java
create mode 100644 sdks/java/io/singlestore/build.gradle
create mode 100644 sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/SingleStoreIO.java
create mode 100644 sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/SingleStoreUtil.java
copy sdks/java/io/{google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PublisherOptions.java => singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/package-info.java} (50%)
create mode 100644 sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/DataSourceConfigurationTest.java
create mode 100644 sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/ReadTest.java
create mode 100644 sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/ReadWithPartitionsTest.java
create mode 100644 sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/SingleStoreIOIT.java
create mode 100644 sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/SingleStoreIOTestPipelineOptions.java
create mode 100644 sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/SingleStoreUtilTest.java
create mode 100644 sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/TestHelper.java
create mode 100644 sdks/java/io/singlestore/src/test/java/org/apache/beam/sdk/io/singlestore/WriteTest.java
create mode 100644 sdks/python/apache_beam/testing/benchmarks/inference/README.md
create mode 100644 website/www/site/content/en/blog/beam-2.43.0.md