You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2022/02/18 03:40:22 UTC
[beam] branch nightly-refs/heads/master updated (895a1dd -> c1525a4)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 895a1dd Bump dataflow.fnapi_container_version (#16874)
add c577f4a Fix TableRow conversion for the case of fields named "f"
add a78fec7 Merge pull request #16872: [BEAM-13959] Fix StorageApi writes when a field is named f
add 2a99d19 Disable AfterSynchronizedProcessingTime test on Dataflow
add 459fea0 Merge pull request #16859: [BEAM-13952] Disable AfterSynchronizedProcessingTime test on Dataflow
add 87dba3a Case studies page improvements (#16702)
add 53f5a3c [BEAM-13563] Introducing common AWS ClientBuilderFactory to unify configuration of sync / async AWS clients across all IOs (AWS Sdk v2). [BEAM-13206] Additionally, retry configuration is pushed down into clients to delegate retries to the AWS Sdk.
add adcde0e Merge pull request #16760: [BEAM-13563] Introducing common AWS ClientBuilderFactory (AWS Sdk v2)
add d84541d [BEAM-13946] Add get_dummies(), a non-deferred column operation on categorical columns (#16615)
add 45170d0 Fix final allowskew error to properly handle a large allowedSkew
add 6c04c3c Merge pull request #16885: Fix final allowskew error to properly handle a large allowedSkew
add a47bfaa Simplify README for new users
add c1525a4 Merge pull request #16814 from Simplify README for new users
No new revisions were added by this update.
Summary of changes:
CHANGES.md | 3 +
README.md | 12 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 1 +
runners/google-cloud-dataflow-java/build.gradle | 3 +
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 4 +-
sdks/java/io/amazon-web-services2/build.gradle | 2 +-
.../sdk/io/aws2/common/ClientBuilderFactory.java | 250 +++++++++++
.../sdk/io/aws2/common/ClientConfiguration.java | 189 +++++++++
.../io/aws2/common/HttpClientConfiguration.java | 170 ++++++++
.../sdk/io/aws2/common/RetryConfiguration.java | 118 ++++++
.../io/aws2/{kinesis => common}/package-info.java | 4 +-
.../aws2/dynamodb/BasicDynamoDbClientProvider.java | 48 +--
.../beam/sdk/io/aws2/dynamodb/DynamoDBIO.java | 464 +++++++++++----------
.../io/aws2/dynamodb/DynamoDbClientProvider.java | 5 +
.../sdk/io/aws2/kinesis/AWSClientsProvider.java | 6 +-
.../sdk/io/aws2/kinesis/BasicKinesisProvider.java | 61 +--
.../apache/beam/sdk/io/aws2/kinesis/KinesisIO.java | 276 ++++--------
.../beam/sdk/io/aws2/kinesis/KinesisSource.java | 29 +-
.../io/aws2/kinesis/SimplifiedKinesisClient.java | 7 +-
.../apache/beam/sdk/io/aws2/options/AwsModule.java | 161 +++----
.../beam/sdk/io/aws2/options/AwsOptions.java | 149 ++++---
.../sdk/io/aws2/options/AwsSerializableUtils.java | 35 +-
.../io/aws2/s3/DefaultS3ClientBuilderFactory.java | 24 +-
.../io/aws2/sns/BasicSnsAsyncClientProvider.java | 39 +-
.../sdk/io/aws2/sns/BasicSnsClientProvider.java | 48 +--
.../beam/sdk/io/aws2/sns/SnsClientProvider.java | 5 +
.../org/apache/beam/sdk/io/aws2/sns/SnsIO.java | 314 ++++++--------
.../sdk/io/aws2/sqs/BasicSqsClientProvider.java | 48 +--
.../beam/sdk/io/aws2/sqs/SqsClientProvider.java | 5 +
.../org/apache/beam/sdk/io/aws2/sqs/SqsIO.java | 219 ++++++----
.../beam/sdk/io/aws2/sqs/SqsUnboundedReader.java | 26 +-
.../beam/sdk/io/aws2/sqs/SqsUnboundedSource.java | 3 +-
.../org/apache/beam/sdk/io/aws2/ITEnvironment.java | 9 +-
.../beam/sdk/io/aws2/MockClientBuilderFactory.java | 96 +++++
.../io/aws2/common/ClientBuilderFactoryTest.java | 398 ++++++++++++++++++
.../ClientConfigurationTest.java} | 37 +-
.../sdk/io/aws2/common/RetryConfigurationTest.java | 50 +++
.../beam/sdk/io/aws2/dynamodb/DynamoDBIOIT.java | 17 +-
.../sdk/io/aws2/dynamodb/DynamoDBIOReadTest.java | 126 ++++--
.../sdk/io/aws2/dynamodb/DynamoDBIOWriteTest.java | 164 ++++----
.../sdk/io/aws2/kinesis/AmazonKinesisMock.java | 397 ------------------
.../beam/sdk/io/aws2/kinesis/KinesisIOIT.java | 9 +-
.../sdk/io/aws2/kinesis/KinesisIOReadTest.java | 234 +++++++++--
.../sdk/io/aws2/kinesis/KinesisMockReadTest.java | 110 -----
.../sdk/io/aws2/kinesis/KinesisServiceMock.java | 84 ----
.../beam/sdk/io/aws2/options/AwsModuleTest.java | 66 +--
.../beam/sdk/io/aws2/options/AwsOptionsTest.java | 73 ++--
.../aws2/s3/DefaultS3ClientBuilderFactoryTest.java | 24 +-
.../beam/sdk/io/aws2/s3/S3ResourceIdTest.java | 3 +-
.../apache/beam/sdk/io/aws2/s3/S3TestUtils.java | 7 +-
.../org/apache/beam/sdk/io/aws2/sns/SnsIOIT.java | 19 +-
.../org/apache/beam/sdk/io/aws2/sns/SnsIOTest.java | 171 +++++---
.../sdk/io/aws2/sns/StaticSnsClientProvider.java | 34 --
.../org/apache/beam/sdk/io/aws2/sqs/SqsIOIT.java | 15 +-
.../apache/beam/sdk/io/aws2/sqs/SqsIOReadTest.java | 164 ++++++++
.../org/apache/beam/sdk/io/aws2/sqs/SqsIOTest.java | 88 ----
.../beam/sdk/io/aws2/sqs/SqsIOWriteTest.java | 108 +++++
.../sdk/io/aws2/sqs/SqsUnboundedReaderTest.java | 19 +-
.../io/gcp/bigquery/TableRowToStorageApiProto.java | 46 +-
.../bigquery/TableRowToStorageApiProtoTest.java | 294 ++++++++++++-
sdks/python/apache_beam/dataframe/frames.py | 40 +-
sdks/python/apache_beam/dataframe/frames_test.py | 103 ++++-
.../apache_beam/dataframe/pandas_doctests_test.py | 2 +
website/ADD_LOGO.md | 57 +++
website/www/site/assets/scss/_case_study.scss | 302 ++++++++++----
.../www/site/content/en/case-studies/dataflow.md | 4 +-
website/www/site/content/en/case-studies/eBay.md | 4 +-
website/www/site/content/en/case-studies/goga.md | 4 +-
website/www/site/content/en/case-studies/hop.md | 10 +-
website/www/site/content/en/case-studies/kio.md | 3 +-
website/www/site/content/en/case-studies/klio.md | 3 +-
.../www/site/content/en/case-studies/mozilla.md | 4 +-
website/www/site/content/en/case-studies/oriel.md | 4 +-
.../www/site/content/en/case-studies/ricardo.md | 19 +-
website/www/site/content/en/case-studies/scio.md | 3 +-
website/www/site/content/en/case-studies/seznam.md | 18 +-
website/www/site/content/en/case-studies/tfx.md | 4 +-
.../www/site/content/en/community/case-study.md | 36 ++
website/www/site/i18n/case-studies/en.yaml | 4 +-
website/www/site/layouts/case-studies/baseof.html | 3 +-
website/www/site/layouts/case-studies/list.html | 156 +++----
.../partials/section-menu/en/community.html | 1 +
.../www/site/static/images/{elipse.svg => add.svg} | 7 +-
.../{arrow-icon_list.svg => arrow-right.svg} | 5 +-
.../www/site/static/images/logo_google_cloud.png | Bin 13665 -> 18190 bytes
85 files changed, 3979 insertions(+), 2407 deletions(-)
create mode 100644 sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientBuilderFactory.java
create mode 100644 sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientConfiguration.java
create mode 100644 sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfiguration.java
create mode 100644 sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/RetryConfiguration.java
copy sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/{kinesis => common}/package-info.java (88%)
create mode 100644 sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/MockClientBuilderFactory.java
create mode 100644 sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/ClientBuilderFactoryTest.java
copy sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/{kinesis/BasicKinesisClientProviderTest.java => common/ClientConfigurationTest.java} (53%)
create mode 100644 sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/RetryConfigurationTest.java
delete mode 100644 sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/AmazonKinesisMock.java
delete mode 100644 sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisMockReadTest.java
delete mode 100644 sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/kinesis/KinesisServiceMock.java
delete mode 100644 sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sns/StaticSnsClientProvider.java
create mode 100644 sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOReadTest.java
delete mode 100644 sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOTest.java
create mode 100644 sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOWriteTest.java
create mode 100644 website/ADD_LOGO.md
create mode 100644 website/www/site/content/en/community/case-study.md
copy website/www/site/static/images/{elipse.svg => add.svg} (59%)
copy website/www/site/static/images/{arrow-icon_list.svg => arrow-right.svg} (74%)