You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2022/01/05 04:18:49 UTC
[beam] branch nightly-refs/heads/master updated (c4fc398 -> 56c2082)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from c4fc398 Merge pull request #16368: [BEAM-13509] Stop sharing SQS client between readers of same source.
add 17b62ad Merge pull request #16404: [BEAM-13586] Fix NPE in DefaultS3ClientBuilderFactory
add 36ce02b [BEAM-13480] Sickbay PubSubIntegrationTest.test_streaming_data_only on Dataflow (#16255)
add 3d272e7 Fix sdk_container_builder too many values to unpack error
add 1636a3a Merge pull request #16417 from y1chi/python_val
add 43a17a3 [BEAM-13430] Swap to use "mainClass" instead of "main" since it was deprecated. (#16400)
add e6c73b4 [BEAM-13430] Replace deprecated "appendix" with "archiveAppendix" (#16401)
add 061533c [BEAM-13591] Bump log4j2 version to 2.17.1
add 5a11778 Merge pull request #16415 from deadwind4/BEAM-13591
add 122ae34 [BEAM-13015] Add jamm as a java agent to the Java SDK harness container (#16412)
add 2d4a17c [BEAM-13430] Partially revert https://github.com/apache/beam/commit/eaacf709a861ea1c260e3856b720266348b1aecc (#16419)
add 7b5c4cd [BEAM-12391] update avro sink to close the opened file handle, instead of getting it by reference from the writer. This ensures that when we load the python writer implementation we don't run into an attribute error
add f63451e Merge pull request #16246: [BEAM-12391] update avro sink to close the opened file handle directly
add 6045ee1 [BEAM-13482] Python fully qualified name external transforms.
add d13c21f Add Python expansion service entry point.
add 0c2f5a5 Merge pull request #16266 [BEAM-13482] Python fully qualified name external transforms.
add 818428f Merge pull request #15863 from [BEAM-13184] Autosharding for JdbcIO.write* transforms
add 9d2fcf3 Add Flink runner support for OrderedListState. This version reads the list entirely into memory, so will not work well for large lists.
add b1341bc Merge pull request #15984: [BEAM-2791] OrderedListState for Flink
add 3220631 [BEAM-11936] Enable FloatingPointAssertionWithinEpsilon errorprone check (#16261)
add 9973c4f [BEAM-11936] Enable LockNotBeforeTry errorprone check (#16259)
add 7eeffe0 [BEAM-11936] Enable errorprone unused checks (#16262)
add 6b68536 Add Nexmark Query 14 (#16337)
add 83e9218 [BEAM-13569] Change Spark dependencies to implementation.
add f0289c0 remove redundant dependency
add e35eea8 remove redundant testImplementation dependencies
add 4c9d4f0 Merge pull request #16396 from ibzib/BEAM-13569
add ea65a05 [BEAM-13015] Migrate all user state and side implementations to support caching. (#16263)
add eea07cf Fix remaining failing perf IT tests.
add 5ccb002 Merge pull request #16418 from y1chi/perf-it-2
add b82e407 [BEAM-13459] Update CHANGES.md, add note about artifact caching for python jobs
add 56c2082 [BEAM-13459] Update CHANGES.md, add note about artifact caching python jobs #16416
No new revisions were added by this update.
Summary of changes:
...Java_PortableValidatesRunner_Spark_Batch.groovy | 2 +
...va_PortableValidatesRunner_Flink_Docker.groovy} | 10 +-
.test-infra/validate-runner/build.gradle | 2 +-
CHANGES.md | 1 +
build.gradle.kts | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 19 +-
examples/java/build.gradle | 4 +-
.../datatokenization/DataTokenization.java | 2 +-
.../examples/complete/datatokenization/README.md | 2 +-
.../complete/kafkatopubsub/KafkaToPubsub.java | 2 +-
.../beam/examples/complete/kafkatopubsub/README.md | 2 +-
.../apache/beam/examples/snippets/Snippets.java | 8 +
examples/java/twitter/build.gradle | 2 +-
.../core/construction/PTransformMatchersTest.java | 21 -
runners/direct-java/build.gradle | 4 +
.../runners/direct/TransformEvaluatorRegistry.java | 16 -
.../runners/direct/CloningBundleFactoryTest.java | 67 ---
runners/flink/flink_runner.gradle | 5 +-
runners/flink/job-server/flink_job_server.gradle | 47 +-
.../flink/FlinkStreamingTransformTranslators.java | 17 -
.../wrappers/streaming/DoFnOperator.java | 2 +-
.../state/FlinkBroadcastStateInternals.java | 131 ------
.../streaming/state/FlinkStateInternals.java | 153 +++++-
.../streaming/ExecutableStageDoFnOperatorTest.java | 1 +
runners/google-cloud-dataflow-java/build.gradle | 4 +-
.../beam/runners/dataflow/DataflowRunner.java | 110 -----
.../runners/dataflow/DataflowPipelineJobTest.java | 25 -
.../fn/data/RemoteGrpcPortWriteOperation.java | 6 +-
.../control/DefaultJobBundleFactory.java | 12 +-
.../fnexecution/control/RemoteExecutionTest.java | 2 -
runners/jet/build.gradle | 2 +
runners/portability/java/build.gradle | 2 +
runners/samza/build.gradle | 2 +
runners/samza/job-server/build.gradle | 45 +-
runners/spark/job-server/spark_job_server.gradle | 192 ++++----
runners/spark/spark_runner.gradle | 18 +-
runners/twister2/build.gradle | 2 +
sdks/java/container/Dockerfile | 3 +
sdks/java/container/boot.go | 9 +
sdks/java/container/build.gradle | 1 +
sdks/java/container/common.gradle | 1 +
sdks/java/core/build.gradle | 1 +
.../org/apache/beam/sdk/schemas/SchemaCoder.java | 20 -
...Metrics.java => UsesSdkHarnessEnvironment.java} | 5 +-
.../apache/beam/sdk/values/PCollectionViews.java | 59 ---
.../apache/beam/sdk/SdkHarnessEnvironmentTest.java | 69 +++
.../apache/beam/sdk/coders/CoderRegistryTest.java | 4 -
.../org/apache/beam/sdk/transforms/ParDoTest.java | 15 +-
.../sdk/transforms/reflect/DoFnSignaturesTest.java | 9 -
.../GrowableOffsetRangeTrackerTest.java | 2 +-
sdks/java/expansion-service/app/build.gradle | 2 +-
sdks/java/expansion-service/build.gradle | 2 +-
.../core/translate/TimestampExtractTransform.java | 8 -
.../schemaio-expansion-service/build.gradle | 2 +-
sdks/java/extensions/sql/build.gradle | 4 +-
sdks/java/extensions/sql/datacatalog/build.gradle | 2 +-
.../extensions/sql/expansion-service/build.gradle | 2 +-
sdks/java/extensions/sql/hcatalog/build.gradle | 2 +-
.../org/apache/beam/sdk/fn/CancellableQueue.java | 4 +-
.../beam/sdk/fn/stream/PrefetchableIterable.java | 3 +
.../beam/sdk/fn/stream/PrefetchableIterables.java | 53 ++-
.../sdk/fn/stream/PrefetchableIterablesTest.java | 20 +
sdks/java/harness/build.gradle | 6 +-
.../java/org/apache/beam/fn/harness/Caches.java | 1 +
.../apache/beam/fn/harness/state/BagUserState.java | 57 ++-
.../beam/fn/harness/state/FnApiStateAccessor.java | 135 +++---
.../beam/fn/harness/state/IterableSideInput.java | 47 +-
.../state/LazyCachingIteratorToIterable.java | 6 +-
.../beam/fn/harness/state/MultimapSideInput.java | 69 +--
.../beam/fn/harness/state/MultimapUserState.java | 184 +++++---
.../fn/harness/state/StateFetchingIterators.java | 9 +-
.../beam/fn/harness/state/BagUserStateTest.java | 170 ++++++-
.../fn/harness/state/IterableSideInputTest.java | 96 ++++
.../fn/harness/state/MultimapSideInputTest.java | 71 ++-
.../fn/harness/state/MultimapUserStateTest.java | 524 +++++++++++++++++----
.../harness/state/StateFetchingIteratorsTest.java | 2 +-
.../io/aws2/s3/DefaultS3ClientBuilderFactory.java | 27 +-
.../aws2/s3/DefaultS3ClientBuilderFactoryTest.java | 92 ++++
.../elasticsearch-tests-5/build.gradle | 2 +-
.../elasticsearch-tests-6/build.gradle | 2 +-
.../elasticsearch-tests-7/build.gradle | 2 +-
.../elasticsearch-tests-common/build.gradle | 2 +-
sdks/java/io/expansion-service/build.gradle | 2 +-
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 24 -
sdks/java/io/hadoop-format/build.gradle | 3 +
.../sdk/io/hadoop/format/TestRowDBWritable.java | 10 -
sdks/java/io/hcatalog/build.gradle | 2 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 107 ++++-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 39 ++
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 30 ++
.../org/apache/beam/sdk/io/xml/XmlSourceTest.java | 4 +-
sdks/java/testing/expansion-service/build.gradle | 4 +-
sdks/java/testing/kafka-service/build.gradle | 4 +-
sdks/java/testing/load-tests/build.gradle | 3 +-
sdks/java/testing/nexmark/build.gradle | 2 +-
.../apache/beam/sdk/nexmark/NexmarkLauncher.java | 3 +
.../apache/beam/sdk/nexmark/NexmarkQueryName.java | 1 +
.../nexmark/queries/{Query9.java => Query14.java} | 22 +-
sdks/java/testing/tpcds/build.gradle | 2 +-
sdks/java/testing/watermarks/build.gradle | 2 +-
sdks/python/apache_beam/io/avroio.py | 25 +-
sdks/python/apache_beam/io/avroio_test.py | 15 +
.../apache_beam/io/gcp/pubsub_integration_test.py | 2 +
.../runners/portability/expansion_service_main.py | 72 +++
.../runners/portability/sdk_container_builder.py | 2 +-
sdks/python/apache_beam/transforms/external.py | 10 +-
.../transforms/fully_qualified_named_transform.py | 107 +++++
.../fully_qualified_named_transform_test.py | 102 ++++
sdks/python/apache_beam/typehints/schemas.py | 21 +-
.../site/content/en/get-started/quickstart-java.md | 2 +-
110 files changed, 2226 insertions(+), 1146 deletions(-)
copy .test-infra/jenkins/{job_PreCommit_Java_PortableValidatesRunner_Flink_Batch.groovy => job_PreCommit_Java_PortableValidatesRunner_Flink_Docker.groovy} (79%)
copy sdks/java/core/src/main/java/org/apache/beam/sdk/testing/{UsesSystemMetrics.java => UsesSdkHarnessEnvironment.java} (86%)
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/SdkHarnessEnvironmentTest.java
create mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/IterableSideInputTest.java
create mode 100644 sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/DefaultS3ClientBuilderFactoryTest.java
copy sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/{Query9.java => Query14.java} (63%)
create mode 100644 sdks/python/apache_beam/runners/portability/expansion_service_main.py
create mode 100644 sdks/python/apache_beam/transforms/fully_qualified_named_transform.py
create mode 100644 sdks/python/apache_beam/transforms/fully_qualified_named_transform_test.py