You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2022/01/27 03:39:22 UTC
[beam] branch nightly-refs/heads/master updated (81006dd -> f72f197)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 81006dd Merge pull request #16594 from [BEAM-13710][Playground] Update the Apache Beam privacy policy with Playground note
add 7b06fa7 [BEAM-8807] Add integration test for SnsIO.write (Sdk v1 & v2)
add 41d585f Merge pull request #16595З: [BEAM-8807] Add integration test for SnsIO.write (Sdk v1 & v2)
add 04a6125 [BEAM-7928] Allow users to specify worker disk type for Dataflow runner (#16622)
add f556c60 [BEAM-13746] Fix deserialization of SSECustomerKey for AWS Sdk v2
add ed2f1fb Merge pull request #16623: [BEAM-13746] Fix deserialization of SSECustomerKey for AWS Sdk v2
add 3c83a3f Merge pull request #16534 from [BEAM-13671][Playground] Add backend contribute guide to the project
add 584940f [BEAM-13271] Bump errorprone to 2.10.0 (#16231)
add 2269b04 [BEAM-13595] Don't load main session when cloudpickle is used. (#16589)
add a159aff [BEAM-13093] Enable JavaUsingPython CrossLanguageValidateRunner test for dataflow runner v2
add f72f197 Merge pull request #15767 from ihji/BEAM-13093
No new revisions were added by this update.
Summary of changes:
.github/PULL_REQUEST_TEMPLATE.md | 12 +-
.test-infra/jenkins/PythonTestProperties.groovy | 1 -
...it_CrossLanguageValidatesRunner_Dataflow.groovy | 62 ---
...alidatesRunner_JavaUsingPython_Dataflow.groovy} | 12 +-
...datesRunner_PythonUsingJavaSQL_Dataflow.groovy} | 19 +-
...alidatesRunner_PythonUsingJava_Dataflow.groovy} | 14 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 53 ++-
.../ReadFromTwitterDoFn.java | 9 +-
.../twitterstreamgenerator/TwitterStream.java | 2 +-
playground/backend/CONTRIBUTE.md | 120 ++++++
.../construction/DeduplicatedFlattenFactory.java | 5 -
.../runners/core/construction/graph/Networks.java | 3 +-
.../graph/TrivialNativeTransformExpander.java | 2 +-
.../core/construction/ValidateRunnerXlangTest.java | 427 ++++++++++++---------
.../beam/runners/core/StatefulDoFnRunner.java | 2 +-
.../beam/runners/core/metrics/CounterCell.java | 2 +-
.../direct/ImmutabilityEnforcementFactory.java | 3 +-
.../runners/flink/metrics/MetricsAccumulator.java | 1 +
.../flink/batch/NonMergingGroupByKeyTest.java | 1 +
runners/google-cloud-dataflow-java/build.gradle | 100 +++--
.../examples/build.gradle | 4 +-
.../beam/runners/dataflow/internal/IsmFormat.java | 5 +-
.../runners/dataflow/worker/graph/Networks.java | 3 +-
.../worker/DataflowOperationContextTest.java | 2 -
.../dataflow/worker/GroupingShuffleReaderTest.java | 1 +
.../dataflow/worker/IsmSideInputReaderTest.java | 4 +-
.../dataflow/worker/WindmillKeyedWorkItemTest.java | 3 -
.../control/DefaultJobBundleFactory.java | 2 +-
.../apache/beam/runners/jet/JetPipelineResult.java | 1 +
.../runners/samza/util/PipelineJsonRenderer.java | 2 +
.../translators/functions/ElemToBytesFunction.java | 2 +-
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 3 +
.../pkg/beam/runners/dataflow/dataflowlib/job.go | 2 +
.../container/license_scripts/dep_urls_java.yaml | 4 +
.../java/org/apache/beam/sdk/coders/AvroCoder.java | 2 +-
.../java/org/apache/beam/sdk/transforms/Regex.java | 6 +-
.../beam/sdk/transforms/display/DisplayData.java | 6 +-
.../org/apache/beam/sdk/util/WindowTracing.java | 4 +-
.../org/apache/beam/sdk/values/ShardedKey.java | 2 +-
.../org/apache/beam/sdk/coders/AvroCoderTest.java | 1 +
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 4 +-
.../apache/beam/sdk/io/GenerateSequenceTest.java | 1 +
.../sdk/io/SerializableAvroCodecFactoryTest.java | 2 +-
.../sdk/options/ProxyInvocationHandlerTest.java | 2 +-
.../apache/beam/sdk/schemas/utils/TestPOJOs.java | 2 +-
.../org/apache/beam/sdk/testing/PAssertTest.java | 16 +-
.../beam/sdk/transforms/join/CoGbkResultTest.java | 4 +-
.../sdk/util/ExposedByteArrayOutputStreamTest.java | 7 +-
.../beam/sdk/util/IdentitySideInputWindowFn.java | 2 +-
.../sdk/extensions/euphoria/core/util/IOUtils.java | 1 +
.../sql/jdbc/BeamSqlLineTestingUtils.java | 8 +-
.../harness/state/StateFetchingIteratorsTest.java | 9 +-
.../sdk/io/aws/dynamodb/AwsClientsProvider.java | 8 +-
.../sdk/io/aws/dynamodb/BasicDynamoDBProvider.java | 15 -
.../beam/sdk/io/aws/sns/AwsClientsProvider.java | 8 +-
.../beam/sdk/io/aws/sns/BasicSnsProvider.java | 16 +-
.../java/org/apache/beam/sdk/io/aws/sns/SnsIO.java | 4 +-
.../org/apache/beam/sdk/io/aws/ITEnvironment.java | 31 +-
.../io/aws/dynamodb/StaticAwsClientsProvider.java | 6 -
.../apache/beam/sdk/io/aws/s3/S3FileSystemIT.java | 12 +-
.../beam/sdk/io/aws/s3/S3ResourceIdTest.java | 7 +-
.../org/apache/beam/sdk/io/aws/sns/SnsIOIT.java | 159 ++++++++
.../org/apache/beam/sdk/io/aws/sns/SnsIOTest.java | 16 +-
.../org/apache/beam/sdk/io/aws/sqs/SqsIOTest.java | 3 +-
.../apache/beam/sdk/io/aws2/options/AwsModule.java | 33 +-
.../apache/beam/sdk/io/aws2/s3/SSECustomerKey.java | 6 +-
.../org/apache/beam/sdk/io/aws2/sns/SnsIO.java | 4 +-
.../org/apache/beam/sdk/io/aws2/ITEnvironment.java | 31 +-
.../beam/sdk/io/aws2/options/AwsModuleTest.java | 11 +
.../org/apache/beam/sdk/io/aws2/sns/SnsIOIT.java | 158 ++++++++
.../org/apache/beam/sdk/io/aws2/sns/SnsIOTest.java | 10 +-
.../org/apache/beam/sdk/io/aws2/sqs/SqsIOTest.java | 3 +-
.../beam/sdk/io/amqp/AmqpMessageCoderTest.java | 2 +-
.../beam/sdk/io/cassandra/CassandraIOTest.java | 6 +-
.../beam/sdk/io/clickhouse/ClickHouseIO.java | 4 +-
.../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 12 +-
.../gcp/bigquery/BigQueryStorageStreamSource.java | 1 -
.../io/gcp/bigquery/BigQueryIOMetadataTest.java | 1 +
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 2 +-
.../sdk/io/hadoop/SerializableConfiguration.java | 6 +-
.../apache/beam/sdk/io/hdfs/HadoopFileSystem.java | 12 +-
.../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 10 +-
.../sdk/io/hadoop/format/HDFSSynchronization.java | 1 -
.../org/apache/beam/sdk/io/kinesis/KinesisIO.java | 9 +-
.../java/org/apache/beam/sdk/io/kudu/KuduIO.java | 4 +-
.../org/apache/beam/sdk/io/parquet/ParquetIO.java | 7 +-
.../apache/beam/sdk/io/parquet/ParquetIOTest.java | 7 +-
.../io/snowflake/data/SnowflakeTableSchema.java | 6 +-
.../beam/sdk/nexmark/model/AuctionCount.java | 2 +-
.../beam/sdk/nexmark/model/AuctionPrice.java | 2 +-
.../org/apache/beam/sdk/nexmark/model/Bid.java | 23 +-
.../beam/sdk/nexmark/model/NameCityStateId.java | 2 +-
.../beam/sdk/nexmark/queries/sql/SqlQuery0.java | 3 +-
.../beam/sdk/nexmark/queries/sql/SqlQuery1.java | 3 +-
.../beam/sdk/nexmark/queries/sql/SqlQuery2.java | 11 +-
.../beam/sdk/nexmark/queries/sql/SqlQuery3.java | 5 +-
.../beam/sdk/nexmark/queries/sql/SqlQuery5.java | 3 +-
.../beam/sdk/nexmark/queries/sql/SqlQuery7.java | 33 +-
.../python/apache_beam/options/pipeline_options.py | 3 +-
.../apache_beam/runners/portability/stager.py | 10 +-
.../apache_beam/runners/portability/stager_test.py | 15 +
.../apache_beam/runners/worker/sdk_worker_main.py | 26 +-
102 files changed, 1131 insertions(+), 608 deletions(-)
delete mode 100644 .test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Dataflow.groovy
copy .test-infra/jenkins/{job_PostCommit_CrossLanguageValidatesRunner_Direct.groovy => job_PostCommit_CrossLanguageValidatesRunner_JavaUsingPython_Dataflow.groovy} (74%)
copy .test-infra/jenkins/{job_PostCommit_CrossLanguageValidatesRunner_Samza.groovy => job_PostCommit_CrossLanguageValidatesRunner_PythonUsingJavaSQL_Dataflow.groovy} (66%)
copy .test-infra/jenkins/{job_PostCommit_CrossLanguageValidatesRunner_Direct.groovy => job_PostCommit_CrossLanguageValidatesRunner_PythonUsingJava_Dataflow.groovy} (72%)
create mode 100644 playground/backend/CONTRIBUTE.md
create mode 100644 sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sns/SnsIOIT.java
create mode 100644 sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sns/SnsIOIT.java