You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2022/02/01 03:34:21 UTC
[beam] branch nightly-refs/heads/master updated (6c9c208 -> 59b0b2a)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 6c9c208 [BEAM-10206] Add Go Vet to Github Actions (#16612)
add 08a5651 Generalize S3FileSystem to support multiple URI schemes.
add e638c11 Merge pull request #16607: [BEAM-13245] Generalize S3FileSystem to support multiple URI schemes.
add b006c5a [BEAM-13769] Skip test_main_session_not_staged_when_using_cloudpickle (#16651)
add c5007de Fix java test
add 60f94d3 Merge pull request #16663 from [BEAM-13773] [Playground] [Bugfix] Error during executing of tests on Java SDK
add 7ecdc52 Change executable name fo go tests
add 71decd8 Merge pull request #16662 from [BEAM-13772][Playground][Bugfix] Change executable name for go tests
add 09690ed [BEAM-6744] Support implicitly setting project id in Go Dataflow runner (#16643)
add 4e5db10 Merge pull request #16493 from [BEAM-13632][Playground] Save catalog data to the cache
add 43a5e5b Exclude jul-to-slf4j from Spark runner in quickstart POM templates (#16656)
add ba62402 [BEAM-11936] Enable a few errorprone checks that were broken by pinned guava in annotationProcessor (#16647)
add 1d89eac [BEAM-13780] Add CONTRIBUTING.md pointing to main guide (#16666)
add f72bd05 Exclude per-key order tests on Twister2 runner
add 89914d7 Merge pull request #16624 from [BEAM-13749] Exclude per-key order tests on Twister2 runner
add db83b75 [BEAM-13777] Accept cache capacity as input parameter instead of default max integer (#16652)
add ca88d23 [BEAM-13051][A] Enable pylint warnings (function-redefined/bad-super-call) (#16521)
add 858258c Support SCIO SDK via sbt projects
add 510417a Merge pull request #16563 from [BEAM-13701][Playground] Support SCIO SDK via sbt projects
add f6ab250 [BEAM-13779] Fix pr labeling (#16665)
add 91738cb Merge pull request #16581 from [BEAM-12164]: Add SpannerIO.readChangeStreams
add 4af05bb Fix labeler trigger (#16674)
add d9d9026 [BEAM-11648] Share thread pool across RetryManager instances.
add b7095c8 Merge pull request #16619: [BEAM-11648] Share thread pool across RetryManager instances.
add 4865ca4 Fix Java SDK container image name for load-tests and nexmark
add 10da5af Merge pull request #16645 from ihji/fix_nexmark_dataflow_v2
add 9b374aa [BEAM-13781] Exclude grpc-netty-shaded from gax-grpc's dependency (#16672)
add c60c3e7 [BEAM-13051] Fixed pylint warnings : raising-non-exception (E0710), super-on-old-class (E1002) (#16541)
add 525920f [BEAM-13740] Correctly install go before running tests (#16673)
add f643bd9 [BEAM-12830] Update local Docker env Go version. (#16670)
add 8f8577a [BEAM-13051][B] Enable pylint warnings (import-error/invalid-unary-operand-type) (#16522)
add 37da20e [BEAM-13430] Revert Spark libraries in Spark runner to provided (#16675)
add c2d4e51 [BEAM-12240] Add Java 17 support (#16568)
add 59b0b2a [BEAM-13760] Add random component to default python dataflow job name (#16641)
No new revisions were added by this update.
Summary of changes:
.github/autolabeler.yml | 126 +++----
.github/workflows/java_tests.yml | 20 +-
.github/workflows/label_prs.yml | 35 ++
.github/workflows/python_tests.yml | 13 +-
.test-infra/jenkins/CommonJobProperties.groovy | 1 +
.test-infra/jenkins/JavaTestProperties.groovy | 2 +-
.test-infra/jenkins/NexmarkBuilder.groovy | 29 +-
CHANGES.md | 1 +
CONTRIBUTING.md | 29 ++
build.gradle.kts | 4 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 23 +-
dev-support/docker/Dockerfile | 2 +-
playground/backend/cmd/server/controller.go | 30 +-
playground/backend/cmd/server/server.go | 21 ++
playground/backend/configs/SDK_SCIO.json | 10 +
playground/backend/containers/scio/Dockerfile | 69 ++++
.../backend/containers/{java => scio}/app.yaml | 0
playground/backend/containers/scio/build.gradle | 74 ++++
playground/backend/containers/scio/entrypoint.sh | 25 ++
.../containers/{java => scio}/settings.gradle | 0
playground/backend/internal/cache/cache.go | 12 +-
.../backend/internal/cache/local/local_cache.go | 19 +
.../internal/cache/local/local_cache_test.go | 171 +++++++++
.../backend/internal/cache/redis/redis_cache.go | 30 ++
.../internal/cache/redis/redis_cache_test.go | 195 +++++++++++
.../cloud_bucket/precompiled_objects_test.go | 22 +-
.../internal/code_processing/code_processing.go | 2 +-
.../internal/environment/environment_service.go | 4 +-
.../environment/environment_service_test.go | 8 +-
playground/backend/internal/executors/executor.go | 2 +-
playground/backend/internal/fs_tool/fs.go | 3 +
playground/backend/internal/fs_tool/python_fs.go | 2 +-
.../internal/fs_tool/{python_fs.go => scio_fs.go} | 10 +-
.../backend/internal/preparers/java_preparers.go | 12 +-
.../backend/internal/preparers/scio_preparers.go | 69 ++++
.../internal/setup_tools/builder/setup_builder.go | 26 +-
.../setup_tools/life_cycle/life_cycle_setuper.go | 50 +++
.../life_cycle/life_cycle_setuper_test.go | 1 -
.../internal/utils/precompiled_objects_utils.go | 53 +++
.../utils/precompiled_objects_utils_test.go | 131 +++++++
.../backend/internal/utils/preparators_utils.go | 2 +
.../backend/internal/utils/validators_utils.go | 2 +
.../backend/internal/validators/scio_validators.go | 50 +++
.../backend/internal/validators/validator.go | 5 +-
playground/backend/new_scio_project.sh | 18 +
.../core/construction/EnvironmentsTest.java | 2 +
runners/google-cloud-dataflow-java/build.gradle | 7 +-
.../examples/build.gradle | 11 +
.../beam/runners/dataflow/DataflowRunner.java | 10 +-
runners/spark/spark_runner.gradle | 14 +-
runners/twister2/build.gradle | 3 +
sdks/go/pkg/beam/options/gcpopts/options.go | 17 +-
sdks/go/pkg/beam/options/gcpopts/options_test.go | 82 +++++
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 2 +-
sdks/java/container/build.gradle | 1 +
.../java/container/java17/build.gradle | 11 +-
.../java/org/apache/beam/fn/harness/Caches.java | 10 +-
...ava => DefaultS3FileSystemSchemeRegistrar.java} | 12 +-
.../apache/beam/sdk/io/aws2/s3/S3FileSystem.java | 88 ++---
.../sdk/io/aws2/s3/S3FileSystemConfiguration.java | 119 +++++++
.../beam/sdk/io/aws2/s3/S3FileSystemRegistrar.java | 22 +-
...strar.java => S3FileSystemSchemeRegistrar.java} | 34 +-
.../io/aws2/s3/S3ReadableSeekableByteChannel.java | 13 +-
.../apache/beam/sdk/io/aws2/s3/S3ResourceId.java | 60 +++-
.../beam/sdk/io/aws2/s3/S3WritableByteChannel.java | 41 ++-
.../beam/sdk/io/aws2/s3/S3FileSystemTest.java | 383 +++++++++++++++++----
.../beam/sdk/io/aws2/s3/S3ResourceIdTest.java | 92 ++++-
.../apache/beam/sdk/io/aws2/s3/S3TestUtils.java | 77 ++++-
.../sdk/io/aws2/s3/S3WritableByteChannelTest.java | 89 ++++-
sdks/java/io/cassandra/build.gradle | 7 +-
.../beam/sdk/io/cassandra/CassandraIOTest.java | 6 +-
sdks/java/io/google-cloud-platform/build.gradle | 5 +-
.../beam/sdk/io/gcp/bigquery/RetryManager.java | 6 +-
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 317 +++++++++++++++--
.../gcp/spanner/changestreams/NameGenerator.java | 47 +++
.../changestreams/dao/PartitionMetadataDao.java | 21 ++
.../dofn/DetectNewPartitionsDoFn.java | 6 +-
.../spanner/changestreams/dofn/InitializeDoFn.java | 98 ++++++
.../dofn/PostProcessingMetricsDoFn.java | 115 +++++++
.../dofn/ReadChangeStreamPartitionDoFn.java | 4 +-
.../mapper/ChangeStreamRecordMapper.java | 3 +
.../spanner/changestreams/NameGeneratorTest.java | 25 +-
sdks/java/io/hadoop-file-system/build.gradle | 7 +-
sdks/java/io/hadoop-format/build.gradle | 7 +-
...onfluentSchemaRegistryDeserializerProvider.java | 29 +-
.../src/main/resources/archetype-resources/pom.xml | 6 +
.../src/main/resources/archetype-resources/pom.xml | 6 +
sdks/java/testing/load-tests/build.gradle | 6 +-
sdks/java/testing/nexmark/build.gradle | 8 +-
sdks/java/testing/test-utils/build.gradle | 11 +
.../testutils/jvmverification/JvmVerification.java | 16 +-
sdks/python/.pylintrc | 5 -
.../apache_beam/examples/snippets/snippets_test.py | 4 +-
.../apache_beam/options/pipeline_options_test.py | 2 +-
.../runners/dataflow/internal/apiclient.py | 13 +-
.../runners/dataflow/internal/apiclient_test.py | 2 +-
.../apache_beam/runners/portability/stager_test.py | 3 +
.../apache_beam/transforms/userstate_test.py | 2 +-
.../apache_beam/typehints/decorators_test.py | 10 +-
.../python/apache_beam/typehints/typehints_test.py | 8 +-
sdks/python/apache_beam/utils/processes_test.py | 12 +-
sdks/python/apache_beam/utils/timestamp_test.py | 4 +-
settings.gradle.kts | 2 +
.../en/documentation/runtime/environments.md | 1 +
104 files changed, 2983 insertions(+), 454 deletions(-)
create mode 100644 .github/workflows/label_prs.yml
create mode 100644 CONTRIBUTING.md
create mode 100644 playground/backend/configs/SDK_SCIO.json
create mode 100644 playground/backend/containers/scio/Dockerfile
copy playground/backend/containers/{java => scio}/app.yaml (100%)
create mode 100644 playground/backend/containers/scio/build.gradle
create mode 100755 playground/backend/containers/scio/entrypoint.sh
copy playground/backend/containers/{java => scio}/settings.gradle (100%)
copy playground/backend/internal/fs_tool/{python_fs.go => scio_fs.go} (72%)
create mode 100644 playground/backend/internal/preparers/scio_preparers.go
create mode 100644 playground/backend/internal/validators/scio_validators.go
create mode 100644 playground/backend/new_scio_project.sh
create mode 100644 sdks/go/pkg/beam/options/gcpopts/options_test.go
copy .test-infra/jenkins/JavaTestProperties.groovy => sdks/java/container/java17/build.gradle (72%)
copy sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/{S3FileSystemRegistrar.java => DefaultS3FileSystemSchemeRegistrar.java} (77%)
create mode 100644 sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/S3FileSystemConfiguration.java
copy sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/{S3FileSystemRegistrar.java => S3FileSystemSchemeRegistrar.java} (54%)
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/NameGenerator.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dofn/InitializeDoFn.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/dofn/PostProcessingMetricsDoFn.java
copy .test-infra/jenkins/JavaTestProperties.groovy => sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/NameGeneratorTest.java (51%)