You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2022/03/24 04:13:30 UTC
[beam] branch nightly-refs/heads/master updated (1b1ccd8 -> 14862cc)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 1b1ccd8 Merge pull request #17155: [BEAM-14149] Disable BigQueryIOStorageWriteIT for general Java post-commit
add 4ab97a5 [BEAM-13015] Avoid repeated weighing of StateKey in StateFetchingIterator cache usage. (#17121)
add 06f3519 Add option to add modules to JDK add-open (#17110)
add 7ef6560 [BEAM-13015] Clarify ownership of the list for state caching across bundles (#16982)
add cdf99be [BEAM-13232] Close clients properly in KinesisSource. Also use lazy init for AWS clients to avoid unnecessary allocations for some use cases.
add 1d6aacf Merge pull request #17140: [BEAM-13232] Close clients properly in KinesisSource.
add 8cda8a2 [BEAM-14134] Optimize memory allocations for various core coders (#17134)
add 0943d85 [BEAM-14129] Restructure PubsubLiteIO Read side to produce smaller bundles (#17125)
add 5605d44 [BEAM-12697] Add primitive field generation from IR to SBE extension (#16762)
add ea624ef [BEAM-14151] Excluding Spanner CDC tests from Dataflow V1 suite
add 9cadeca Merge pull request #17156 from [BEAM-14151] Excluding Spanner CDC tests from Dataflow V1 suite
add a87d088 [BEAM-13976] [Website] update homepage
add 18e0390 [BEAM-13976] [Website] update homepage, add logo
add 539016f [BEAM-13976] [Website] update text
add 45643bb [BEAM-13979] [Website] add Beam Playground
add 52396c6 [BEAM-13976] [Website] delete Beam Playground
add 1946a1d [BEAM-13976] [Website] change navbar css links rules, delete links from mobile
add 427428f [BEAM-13976] [Website] change padding size between the sections
add de673f7 [BEAM-13976] [Website] change title to capital letters
add 55f3365 [BEAM-13976] [Website] change title
add a73792f [BEAM-13976] [Website] change text
add 48db648 Merge branch 'master' into beam-13976-update-homepage
add 4385c67 [BEAM-13976] [Website] add playground sass, change text-align
add 4b60fd2 Merge branch 'master' into beam-13976-update-homepage
add eeeb7d1 [BEAM-13976] [Website] add playground section, add empty line
add 30788b6 [BEAM-13976][Website] Update text for the `Playground` section on the Home page
add e2473d4 Merge branch 'master' into beam-13976-update-homepage
add 2d55a58 Merge pull request #16919 from akvelon/beam-13976-update-homepage
add 0ee7ab0 [BEAM-13976] [Website] Update Community landing page
add f91c13f [BEAM-13979] [Website] update title
add a276400 [BEAM-13977] [Website] delete available-contact-channels on mobile devices
add b726e0f [BEAM-13977] [Website] change title
add 6c5d1be [BEAM-13977] [Website] change text, add capital letters
add 1aa88da Merge pull request #16945 from akvelon/BEAM-13977-update-community-landing-page
add eb3278c [BEAM-13979] [Website] Update Community/Contact us page
add 44e5d10 [BEAM-13979] [Website] delete space
add e9c4614 [BEAM-13979] [Website] change text
add 2bd698f Merge pull request #16949 from akvelon/BEAM-13979-update_community-contact_us
add 26cfb70 [BEAM-14040] [Website] create new page, add link
add c79868e [BEAM-14040] [Website] add io connectors table
add 3608509 Merge branch 'master' into BEAM-14040-add-connectors
add 01411fc [BEAM-14040] [Website] add overflow to css, add table content
add 98470aa [BEAM-14040] [Website] change ✘ for ✔, add license, add br
add cef3b71 [BEAM-14040] [Website] add empty line
add 7d93530 [BEAM-14040] [Website] change td
add 3b1a2d6 Merge branch 'master' into BEAM-14040-add-connectors
add 59cd96d [BEAM-14040] [Website] change links color
add 9c7cc04 Merge branch 'master' into BEAM-14040-add-connectors
add d39c4ae [BEAM-14040] [Website] change links, change specific version to current
add d5a7816 [BEAM-14040][Website] Update link for the ParquetIO Connector
add 1e2308e Merge pull request #17040 from akvelon/BEAM-14040-add-connectors
add 06f16ef [BEAM-14041] [Website] update built io transforms
add 7df9af8 Merge branch 'master' into BEAM-14041-update-builtin-IO
add 615e04b [BEAM-14041] [Website] move connectors from Miscellaneous to Database
add 87a743d Merge branch 'master' into BEAM-14041-update-builtin-IO
add e697863 [BEAM-14041] [Website] change IO from go to java
add 2f785ea [BEAM-14041][Website] Update link for Pub/Sub Lite, InfluxDB and Firestore IO Connectors
add 00494d4 Merge branch 'master' into BEAM-14041-update-builtin-IO
add c2af980 Merge pull request #17075 from akvelon/BEAM-14041-update-builtin-IO
add c5293ee Updated metrics' CronJob API to use the latest batch version.
add 941b073 Updated cluster to test in metrics-upgrade-clone in BeamMetrics_Publish test
add ac816a92 Updated StateFulSet k8s obejct in cassandra-svc-statefulset.yaml file in order to get the apps/v1 version works.
add cf2437b Updated documentation including cluster specs.
add 5b9eb64 Beam 13058 k8s apis upgrade - elasticsearch (#18)
add b9a2e3c Removed code used for testing.
add 58789c9 Removed code used for testing in job_PostCommit_BeamMetrics_Publish task.
add fa461c4 Beam 13058 k8s apis upgrade - Adding Basic Auth details in documentation (#19)
add 388281e Merge branch 'apache:master' into beam-13058-k8s-apis-upgrade
add 1c46006 Merge branch 'apache:master' into beam-13058-k8s-apis-upgrade
add 0039572 Added comments in initContainers and remove unused code in elasticsearch k8s files.
add 7f911de Merge pull request #17160 from dannymartinm/beam-13058-k8s-apis-upgrade
add 428cd4f [BEAM-13889] Add test cases to jsonx package (#17124)
add b61436e Remove unreachable code in container.go (#17166)
add b9846fb Add ability to handle streaming input to AvroSchemaIOProvider (#17126)
add dae7215 [BEAM-12898] Flink Load Tests failure- UncheckedExecutionException - leaking vms (#17046)
add 14862cc Moving to 2.39.0-SNAPSHOT on master branch.
No new revisions were added by this update.
Summary of changes:
.test-infra/dataproc/flink_cluster.sh | 7 +-
.../jenkins/job_LoadTests_Combine_Flink_Go.groovy | 4 +-
.../job_LoadTests_Combine_Flink_Python.groovy | 4 +-
.../jenkins/job_LoadTests_GBK_Flink_Go.groovy | 4 +-
.../jenkins/job_LoadTests_ParDo_Flink_Go.groovy | 4 +-
.../job_LoadTests_ParDo_Flink_Python.groovy | 8 +-
.../job_LoadTests_SideInput_Flink_Go.groovy | 4 +-
.../jenkins/job_LoadTests_coGBK_Flink_Go.groovy | 4 +-
.test-infra/kubernetes/README.md | 44 +
.../LargeITCluster/cassandra-svc-statefulset.yaml | 10 +-
.../es-services-deployments.yaml | 80 +-
.../SmallITCluster/elasticsearch-svc-rc.yaml | 26 +-
.test-infra/metrics/README.md | 18 +
.../kubernetes/beam-influxdb-autobackup.yaml | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 12 +-
gradle.properties | 4 +-
runners/google-cloud-dataflow-java/build.gradle | 2 +
.../dataflow/options/DataflowPipelineOptions.java | 18 +
sdks/go/container/boot.go | 2 -
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/go/pkg/beam/core/util/jsonx/jsonx_test.go | 86 ++
sdks/java/container/Dockerfile | 5 +
sdks/java/container/boot.go | 149 +++-
sdks/java/container/common.gradle | 8 +
sdks/java/container/java17/option-jamm.json | 13 +
.../beam/sdk/coders/BigEndianIntegerCoder.java | 6 +-
.../apache/beam/sdk/coders/BigEndianLongCoder.java | 6 +-
.../beam/sdk/coders/BigEndianShortCoder.java | 6 +-
.../org/apache/beam/sdk/coders/BitConverters.java | 77 ++
.../org/apache/beam/sdk/coders/DoubleCoder.java | 3 +-
.../org/apache/beam/sdk/coders/FloatCoder.java | 3 +-
.../org/apache/beam/sdk/coders/InstantCoder.java | 6 +-
.../apache/beam/sdk/coders/IterableLikeCoder.java | 12 +-
.../java/org/apache/beam/sdk/coders/MapCoder.java | 9 +-
.../apache/beam/sdk/io/AvroSchemaIOProvider.java | 44 +-
.../apache/beam/sdk/options/SdkHarnessOptions.java | 19 +
.../beam/sdk/io/AvroSchemaIOProviderTest.java | 173 ++++
sdks/java/extensions/sbe/build.gradle | 3 +
.../beam/sdk/extensions/sbe/IrFieldGenerator.java | 146 ++++
.../beam/sdk/extensions/sbe/PrimitiveSbeField.java | 160 ++++
.../apache/beam/sdk/extensions/sbe/SbeField.java | 57 ++
.../apache/beam/sdk/extensions/sbe/SbeSchema.java | 161 ++++
.../beam/sdk/extensions/sbe/SerializableIr.java | 97 +++
.../beam/sdk/extensions/sbe/UnsignedOptions.java | 154 ++++
.../sdk/extensions/sbe/IrFieldGeneratorTest.java | 99 +++
.../sdk/extensions/sbe/PrimitiveSbeFieldTest.java | 110 +++
.../beam/sdk/extensions/sbe/SbeSchemaTest.java | 70 ++
.../sdk/extensions/sbe/SerializableIrTest.java | 93 ++
.../beam/sdk/extensions/sbe/TestSchemas.java | 147 ++++
.../resources/only-primitives-multi-message.xml | 45 +
.../sbe/src/test/resources/only-primitives.xml | 40 +
.../extensions/sbe/src/test/resources/person.xml | 51 ++
.../java/org/apache/beam/fn/harness/Caches.java | 1 -
.../apache/beam/fn/harness/state/BagUserState.java | 2 +
.../beam/fn/harness/state/MultimapUserState.java | 5 +-
.../fn/harness/state/StateFetchingIterators.java | 51 +-
.../harness/state/StateFetchingIteratorsTest.java | 12 +-
.../beam/sdk/io/aws2/kinesis/KinesisReader.java | 10 +-
.../beam/sdk/io/aws2/kinesis/KinesisSource.java | 30 +-
.../io/aws2/kinesis/SimplifiedKinesisClient.java | 75 +-
.../aws2/kinesis/SimplifiedKinesisClientTest.java | 47 +-
.../internal/MemoryBufferedSubscriber.java | 55 ++
.../internal/MemoryBufferedSubscriberImpl.java | 173 ++++
...erverPublisherCache.java => MemoryLimiter.java} | 21 +-
.../gcp/pubsublite/internal/MemoryLimiterImpl.java | 92 ++
.../internal/OffsetByteRangeTracker.java | 104 +--
.../internal/PerServerPublisherCache.java | 7 +-
...herCache.java => PerServerSubscriberCache.java} | 14 +-
.../internal/PerSubscriptionPartitionSdf.java | 21 +-
.../io/gcp/pubsublite/internal/PubsubLiteSink.java | 4 +-
.../{PublisherCache.java => ServiceCache.java} | 54 +-
.../pubsublite/internal/SubscribeTransform.java | 50 +-
.../pubsublite/internal/SubscriberAssembler.java | 1 +
.../SubscriptionPartitionProcessorImpl.java | 180 ++--
.../internal/MemoryBufferedSubscriberImplTest.java | 200 +++++
.../internal/OffsetByteRangeTrackerTest.java | 37 +-
.../SubscriptionPartitionProcessorImplTest.java | 180 ++--
sdks/python/apache_beam/version.py | 2 +-
website/www/site/assets/scss/_connectors.sass | 28 +
website/www/site/assets/scss/_ctas.sass | 3 +-
website/www/site/assets/scss/_global.sass | 3 -
website/www/site/assets/scss/_graphic.scss | 4 +
website/www/site/assets/scss/_hero.scss | 2 +-
website/www/site/assets/scss/_navbar-mobile.sass | 11 +-
website/www/site/assets/scss/_page-nav.scss | 7 +-
website/www/site/assets/scss/_pillars.scss | 2 +-
website/www/site/assets/scss/_quotes.scss | 2 +-
website/www/site/assets/scss/main.scss | 1 +
.../www/site/content/en/community/contact-us.md | 7 +-
.../site/content/en/documentation/io/built-in.md | 16 +-
.../site/content/en/documentation/io/connectors.md | 940 +++++++++++++++++++++
website/www/site/data/en/community_list.yaml | 8 +-
website/www/site/data/en/contact_us.yaml | 19 +-
website/www/site/data/en/graphic.yaml | 10 +-
website/www/site/data/en/pillars.yaml | 8 +-
website/www/site/data/io_matrix.yaml | 24 +
website/www/site/i18n/home/calendar/en.yaml | 2 +-
website/www/site/i18n/home/en.yaml | 6 +-
website/www/site/i18n/home/hero/en.yaml | 6 +-
website/www/site/i18n/home/logos/en.yaml | 2 +-
website/www/site/i18n/home/pillars/en.yaml | 5 +-
website/www/site/i18n/home/quotes/en.yaml | 2 +-
website/www/site/layouts/_default/baseof.html | 6 +-
website/www/site/layouts/community/baseof.html | 1 -
website/www/site/layouts/index.html | 7 +-
.../partials/section-menu/en/documentation.html | 1 +
.../static/images/logos/runners/amazon_kinesis.png | Bin 0 -> 11112 bytes
107 files changed, 4193 insertions(+), 685 deletions(-)
create mode 100644 .test-infra/kubernetes/README.md
create mode 100644 sdks/go/pkg/beam/core/util/jsonx/jsonx_test.go
create mode 100644 sdks/java/container/java17/option-jamm.json
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/coders/BitConverters.java
create mode 100644 sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroSchemaIOProviderTest.java
create mode 100644 sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/IrFieldGenerator.java
create mode 100644 sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/PrimitiveSbeField.java
create mode 100644 sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/SbeField.java
create mode 100644 sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/SbeSchema.java
create mode 100644 sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/SerializableIr.java
create mode 100644 sdks/java/extensions/sbe/src/main/java/org/apache/beam/sdk/extensions/sbe/UnsignedOptions.java
create mode 100644 sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/IrFieldGeneratorTest.java
create mode 100644 sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/PrimitiveSbeFieldTest.java
create mode 100644 sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/SbeSchemaTest.java
create mode 100644 sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/SerializableIrTest.java
create mode 100644 sdks/java/extensions/sbe/src/test/java/org/apache/beam/sdk/extensions/sbe/TestSchemas.java
create mode 100644 sdks/java/extensions/sbe/src/test/resources/only-primitives-multi-message.xml
create mode 100644 sdks/java/extensions/sbe/src/test/resources/only-primitives.xml
create mode 100644 sdks/java/extensions/sbe/src/test/resources/person.xml
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryBufferedSubscriber.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryBufferedSubscriberImpl.java
copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/{PerServerPublisherCache.java => MemoryLimiter.java} (65%)
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryLimiterImpl.java
copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/{PerServerPublisherCache.java => PerServerSubscriberCache.java} (66%)
rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/{PublisherCache.java => ServiceCache.java} (55%)
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/MemoryBufferedSubscriberImplTest.java
create mode 100644 website/www/site/assets/scss/_connectors.sass
create mode 100644 website/www/site/content/en/documentation/io/connectors.md
create mode 100644 website/www/site/static/images/logos/runners/amazon_kinesis.png