You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2020/08/21 02:50:39 UTC
[beam] tag nightly-master updated (d005a5e -> cfa448d)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from d005a5e (commit)
to cfa448d (commit)
from d005a5e Merge pull request #12614 [BEAM-9980] Configure Python versions for direct test suite tasks via settings.gradle
add 41b849e Add Filter lesson to Go SDK katas
add 8046408 Reformat code for readability
add 6c57a61 Change license info format in task-info.yaml
add a6f51c0 Update stepik course
add 614425d Merge pull request #12613 [BEAM-9680] Add Filter with filter package lesson to Go SDK Katas
add f7b49b6 Merge pull request #12581 from [BEAM-10378] Add Azure Blob Storage Filesystem
add 2ca23d2 [BEAM-10674] Python CoGBK: Remove unnecessary step. Adjust parameters to Java version of the test
add efd8a5d [BEAM-10674] Python CoGBK: Add streaming job
add 017936f Merge pull request #12542: [BEAM-10674] Add Python CoGBK load test for streaming on Dataflow
add 1cdb8f4 [BEAM-10292] LocalFileSystem.matchNewResource method throws IllegalArgumentException when the first 'singleResourceSpec' parameter ends with filesystem separator and the second provided parameter indicates that it isn't directory. Fixed due to DefaultFilenamePolicy.ParamsCoder was not able to decode directory on the local file system because it calls FileBasedSink.convertToFileResourceIfPossible method which internally calls FileSystem.matchNewResource method. Other file [...]
add a20b613 [BEAM-10292] changes after review - adding file separator if it doesn't exist if isDirectory == true in LocalFileSystem.matchNewResource - dropped isDirectory field in LocalResourceId - using CoderUtils.clone method in DefaultFilenamePolicyTest.testParamsCoder test
add d86eccd Merge pull request #12050 from davidak09/BEAM-10292
add 899a03e [BEAM-10770] Remove DataflowPortabilityApiUnsupported annotation.
add c7e8c6b Merge pull request #12642 from ibzib/BEAM-10770
add 6db1cae [BEAM-9118] Increase portable_runner_test timeout.
add d20f6fa Merge pull request #12633 from ibzib/BEAM-9118
add 132243b [BEAM-10701] Generate Codecov XML (#12566)
add 02f78d3 [BEAM-10289] Fixing miscalculated SDF split fraction.
add d785650 Merge pull request #12627: [BEAM-10289] Fixing miscalculated SDF split fraction.
add 60d7e91 [BEAM-3301] Consolidating starcgen method names in graph/fn.go and adding SDF names.
add a2a5728 Merge pull request #12628: [BEAM-3301] Consolidating starcgen method names in graph/fn.go and adding SDF names.
add 3a99d21 [BEAM-9979] Fix read index being reported to be reset after any metrics request could happen for the prior bundle and before any metrics request for the next bundle. (#12639)
add cfa448d [BEAM-9546] DataframeTransform can now consume a schema-aware PCollection (#11980)
No new revisions were added by this update.
Summary of changes:
.../job_LoadTests_coGBK_Flink_Python.groovy | 12 +-
.../jenkins/job_LoadTests_coGBK_Python.groovy | 81 ++--
build.gradle | 1 +
.../filter/{pardo => filter}/cmd/main.go | 3 +-
.../filter/filter}/pkg/task/task.go | 7 +-
.../filter/filter}/task-info.yaml | 8 +-
.../filter/filter/task-remote-info.yaml | 2 +
.../common_transforms/filter/filter}/task.md | 9 +-
.../filter/{pardo => filter}/test/task_test.go | 6 +-
.../go/common_transforms/filter/lesson-info.yaml | 1 +
learning/katas/go/course-remote-info.yaml | 2 +-
runners/google-cloud-dataflow-java/build.gradle | 4 +-
sdks/go/pkg/beam/core/graph/fn.go | 26 ++
sdks/go/pkg/beam/core/runtime/exec/datasource.go | 29 +-
.../pkg/beam/core/runtime/exec/datasource_test.go | 6 +-
sdks/go/pkg/beam/util/starcgenx/starcgenx.go | 18 +-
.../apache/beam/sdk/io/DefaultFilenamePolicy.java | 14 +-
.../org/apache/beam/sdk/io/LocalFileSystem.java | 10 +
.../org/apache/beam/sdk/io/LocalResourceId.java | 15 +-
.../testing/DataflowPortabilityApiUnsupported.java | 24 --
.../org/apache/beam/sdk/io/CountingSourceTest.java | 7 +-
.../beam/sdk/io/DefaultFilenamePolicyTest.java | 56 +++
.../apache/beam/sdk/io/LocalFileSystemTest.java | 9 +-
.../org/apache/beam/sdk/metrics/MetricsTest.java | 8 +-
.../org/apache/beam/sdk/testing/PAssertTest.java | 6 +-
.../apache/beam/sdk/transforms/CombineFnsTest.java | 7 +-
.../apache/beam/sdk/transforms/CombineTest.java | 31 +-
.../apache/beam/sdk/transforms/GroupByKeyTest.java | 3 +-
.../beam/sdk/transforms/ParDoSchemaTest.java | 7 +-
.../org/apache/beam/sdk/transforms/ParDoTest.java | 122 +-----
.../beam/sdk/transforms/SplittableDoFnTest.java | 7 +-
.../org/apache/beam/sdk/transforms/ViewTest.java | 13 +-
.../beam/sdk/transforms/windowing/WindowTest.java | 3 +-
.../beam/fn/harness/BeamFnDataReadRunner.java | 25 +-
.../beam/fn/harness/BeamFnDataReadRunnerTest.java | 37 +-
sdks/java/io/azure/build.gradle | 19 +-
.../sdk/io/azure/blobstore/AzfsResourceId.java | 62 ++-
.../azure/blobstore/AzureBlobStoreFileSystem.java | 441 ++++++++++++++++++++-
.../AzureBlobStoreFileSystemRegistrar.java | 10 +-
.../AzureReadableSeekableByteChannel.java | 120 ++++++
.../DefaultBlobstoreClientBuilderFactory.java | 79 ++++
.../options/AzurePipelineOptionsRegistrar.java} | 9 +-
.../options/BlobstoreClientBuilderFactory.java | 7 +-
.../sdk/io/azure/options/BlobstoreOptions.java | 134 +++++++
.../azure/{blobstore => options}/package-info.java | 4 +-
.../sdk/io/azure/blobstore/AzfsResourceIdTest.java | 25 +-
.../blobstore/AzureBlobStoreFileSystemTest.java | 352 ++++++++++++++++
.../java/io/azure/src/test/resources/in.txt | 13 +-
.../org.mockito.plugins.MockMaker | 0
sdks/python/apache_beam/coders/row_coder.py | 17 +-
sdks/python/apache_beam/dataframe/convert.py | 13 +-
sdks/python/apache_beam/dataframe/schemas.py | 71 ++++
sdks/python/apache_beam/dataframe/schemas_test.py | 111 ++++++
sdks/python/apache_beam/dataframe/transforms.py | 17 +-
.../apache_beam/dataframe/transforms_test.py | 151 +++++--
sdks/python/apache_beam/pvalue.py | 2 +-
.../runners/portability/portable_runner_test.py | 2 +-
.../testing/load_tests/co_group_by_key_test.py | 3 -
sdks/python/apache_beam/typehints/schemas.py | 28 ++
sdks/python/scripts/run_pytest.sh | 6 +-
sdks/python/tox.ini | 2 +-
61 files changed, 1899 insertions(+), 418 deletions(-)
copy learning/katas/go/common_transforms/filter/{pardo => filter}/cmd/main.go (94%)
copy learning/katas/go/{core_transforms/combine/simple_function => common_transforms/filter/filter}/pkg/task/task.go (86%)
copy learning/katas/go/{core_transforms/cogroupbykey/cogroupbykey => common_transforms/filter/filter}/task-info.yaml (96%)
create mode 100644 learning/katas/go/common_transforms/filter/filter/task-remote-info.yaml
copy learning/katas/{java/Common Transforms/Filter/Filter => go/common_transforms/filter/filter}/task.md (63%)
copy learning/katas/go/common_transforms/filter/{pardo => filter}/test/task_test.go (90%)
delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/testing/DataflowPortabilityApiUnsupported.java
create mode 100644 sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/AzureReadableSeekableByteChannel.java
create mode 100644 sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/blobstore/DefaultBlobstoreClientBuilderFactory.java
copy sdks/java/io/{amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsPipelineOptionsRegistrar.java => azure/src/main/java/org/apache/beam/sdk/io/azure/options/AzurePipelineOptionsRegistrar.java} (84%)
copy learning/katas/java/Core Transforms/DoFn Additional Parameters/DoFn Additional Parameters/test/org/apache/beam/learning/katas/coretransforms/dofnadditionalparams/TaskTest.java => sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/BlobstoreClientBuilderFactory.java (73%)
create mode 100644 sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/options/BlobstoreOptions.java
copy sdks/java/io/azure/src/main/java/org/apache/beam/sdk/io/azure/{blobstore => options}/package-info.java (89%)
create mode 100644 sdks/java/io/azure/src/test/java/org/apache/beam/sdk/io/azure/blobstore/AzureBlobStoreFileSystemTest.java
copy .test-infra/jenkins/job_PreCommit_RAT.groovy => sdks/java/io/azure/src/test/resources/in.txt (79%)
copy sdks/java/{extensions/ml => io/azure}/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker (100%)
create mode 100644 sdks/python/apache_beam/dataframe/schemas.py
create mode 100644 sdks/python/apache_beam/dataframe/schemas_test.py