You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2022/02/03 03:33:33 UTC
[beam] branch nightly-refs/heads/master updated (b9a7695 -> 51e0e4e)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from b9a7695 [BEAM-13574] Large Wordcount (#16455)
add b4779eb [BEAM-13293] Refactor JDBC IO Go Wrapper (#16686)
add b9c4919 Edit license script for Java, add manual licenses for xz (#16692)
add 1663d6d [BEAM-13563] Restructure Kinesis Source for Aws 2 internally to prepare for ClientBuilderFactory
add cc0b2c5 Merge pull request #16657: [BEAM-13563] Restructure Kinesis Source for AWS 2
add 716e3b1 [BEAM-4665] Allow joining a running dataflow pipeline without throwing (#16689)
add 0701655 [BEAM-13801] Add standard coder tests for state backed iterable. (#16696)
add 74e3e1c [BEAM-13768] Fix NullPointerException in BigQueryStorageSourceBase
add 86dab07 Merge pull request #16644: [BEAM-13768] Fix NullPointerException in BigQueryStorageSourceBase
add 6e2409f [BEAM-13430] Fix provided configuration by removing extendsFrom for implementation (#16688)
add 5193133 [BEAM-12830] Print clearer go version fail message (#16693)
add fc735fe Use ThreadLocal for DESERIALIZATION_CONTEXT (#16680)
add bb0083f Add reference to Books to Learning Resources in website
add 16355ca Merge pull request #16682: Add reference to Books to Learning Resources in website
add 92d12d6 Minor: Add apt update after adding deadsnakes repository in dev env (#16708)
add 7d3af58 Make num-stages counter into an internal counter.
add c7a7eec Merge pull request #16638 from robertwb/internal-counters
add 76d1500 Avoid packaging avro in the java harness jar.
add 303dd12 Merge pull request #16694 [BEAM-13800] Avoid packaging avro in the java harness jar.
add a1a344a [BEAM-13807] Regenerate container images to get TF 2.8.0 (#16707)
add 6c3d0ab [BEAM-13399, BEAM-13683] Eagerly materialize artifacts for automated expansion service, add feature to SQL transform (#16671)
add 5a60dbc Merge pull request #16617 from [BEAM-13743] [Playground] Add context line for examples
add 8853944 Merge pull request #16618 from [BEAM-13744] [Playground] Add context line field to the tags
add 0b3946b Merge pull request #16698 from [BEAM-13802][Playground] [Bugfix] Clean all build directroies, clean kubernetes env
add 0152d50 [BEAM-13293][BEAM-13806] Pipe a SchemaIO flag through Go integration test script. (#16705)
add 02460e2 [BEAM-13737][Playground] Update logic of receiving precompiled objects; Increase timeout;
add 0c587e3 Merge pull request #16620 from [BEAM-13737][Playground] Local retrieving of example objects may fail with context deadline
add 51e0e4e [BEAM-13605] Modify groupby.apply implementation in preparation for pandas 1.4.0 (#16706)
No new revisions were added by this update.
Summary of changes:
.github/workflows/build_playground_backend.yml | 9 +-
.github/workflows/build_playground_frontend.yml | 4 +-
.github/workflows/playground_deploy_examples.yml | 3 +
build.gradle.kts | 6 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 3 +-
dev-support/docker/Dockerfile | 3 +-
.../apache/beam/examples/DebuggingWordCount.java | 1 +
.../org/apache/beam/examples/MinimalWordCount.java | 1 +
.../java/org/apache/beam/examples/WordCount.java | 1 +
.../beam/examples/cookbook/DistinctExample.java | 1 +
.../apache/beam/examples/complete/TfIdfTest.java | 1 +
.../beam/examples/complete/game/GameStatsTest.java | 1 +
.../examples/cookbook/DistinctExampleTest.java | 1 +
.../commontransforms/aggregation/count/Task.java | 1 +
.../commontransforms/aggregation/max/Task.java | 1 +
.../commontransforms/aggregation/mean/Task.java | 1 +
.../commontransforms/aggregation/min/Task.java | 1 +
.../commontransforms/aggregation/sum/Task.java | 1 +
.../katas/commontransforms/filter/filter/Task.java | 1 +
.../katas/commontransforms/filter/pardo/Task.java | 1 +
.../katas/commontransforms/withkeys/Task.java | 1 +
.../katas/coretransforms/branching/Task.java | 1 +
.../combine/binarycombinefnlambda/Task.java | 1 +
.../combine/binarycombinefn/Task.java | 1 +
.../coretransforms/combine/combineperkey/Task.java | 1 +
.../coretransforms/combine/combinefn/Task.java | 1 +
.../katas/coretransforms/combine/simple/Task.java | 1 +
.../katas/coretransforms/composite/Task.java | 1 +
.../katas/coretransforms/flatten/Task.java | 1 +
.../katas/coretransforms/groupbykey/Task.java | 1 +
.../katas/coretransforms/map/mapelements/Task.java | 1 +
.../coretransforms/map/pardoonetomany/Task.java | 1 +
.../katas/coretransforms/map/pardo/Task.java | 1 +
.../katas/coretransforms/partition/Task.java | 1 +
.../katas/coretransforms/sideoutput/Task.java | 1 +
.../learning/katas/examples/wordcount/Task.java | 1 +
.../beam/learning/katas/intro/hello/Task.java | 1 +
.../learning/katas/windowing/fixedwindow/Task.java | 1 +
.../Common Transforms/Aggregation/Count/task.py | 1 +
.../Common Transforms/Aggregation/Largest/task.py | 1 +
.../Common Transforms/Aggregation/Mean/task.py | 1 +
.../Common Transforms/Aggregation/Smallest/task.py | 1 +
.../Common Transforms/Aggregation/Sum/task.py | 1 +
.../python/Common Transforms/Filter/Filter/task.py | 1 +
.../python/Common Transforms/Filter/ParDo/task.py | 1 +
.../Common Transforms/WithKeys/WithKeys/task.py | 1 +
.../Core Transforms/Branching/Branching/task.py | 1 +
.../CoGroupByKey/CoGroupByKey/task.py | 1 +
.../Core Transforms/Combine/Combine PerKey/task.py | 1 +
.../Core Transforms/Combine/CombineFn/task.py | 1 +
.../Combine/Simple Function/task.py | 1 +
.../Composite Transform/task.py | 1 +
.../python/Core Transforms/Flatten/Flatten/task.py | 1 +
.../Core Transforms/GroupByKey/GroupByKey/task.py | 1 +
.../python/Core Transforms/Map/FlatMap/task.py | 1 +
.../katas/python/Core Transforms/Map/Map/task.py | 1 +
.../Core Transforms/Map/ParDo OneToMany/task.py | 1 +
.../katas/python/Core Transforms/Map/ParDo/task.py | 1 +
.../Core Transforms/Partition/Partition/task.py | 1 +
.../Core Transforms/Side Input/Side Input/task.py | 1 +
.../Side Output/Side Output/task.py | 1 +
.../python/Examples/Word Count/Word Count/task.py | 1 +
.../Introduction/Hello Beam/Hello Beam/task.py | 1 +
.../Windowing/Adding Timestamp/ParDo/task.py | 1 +
.../Fixed Time Window/Fixed Time Window/task.py | 1 +
.../beam/model/fnexecution/v1/standard_coders.yaml | 73 ++++
playground/api/v1/api.proto | 1 +
playground/backend/internal/api/v1/api.pb.go | 368 +++++++++++----------
.../internal/cloud_bucket/precompiled_objects.go | 14 +-
.../internal/utils/precompiled_objects_utils.go | 1 +
.../utils/precompiled_objects_utils_test.go | 3 +-
playground/frontend/lib/api/v1/api.pb.dart | 20 ++
playground/frontend/lib/api/v1/api.pbgrpc.dart | 6 +
playground/frontend/lib/api/v1/api.pbjson.dart | 11 +-
playground/infrastructure/config.py | 1 +
playground/infrastructure/helper.py | 18 +-
.../core/construction/ModelCoderRegistrar.java | 49 ++-
.../core/construction/CoderTranslationTest.java | 8 +-
runners/flink/flink_runner.gradle | 1 +
.../worker/legacy-worker/build.gradle | 2 +
.../runners/fnexecution/wire}/CommonCoderTest.java | 81 ++++-
sdks/go/examples/contains/contains.go | 1 +
.../debugging_wordcount/debugging_wordcount.go | 1 +
sdks/go/examples/forest/forest.go | 1 +
sdks/go/examples/grades/grades.go | 1 +
.../minimal_wordcount/minimal_wordcount.go | 1 +
sdks/go/examples/multiout/multiout.go | 1 +
sdks/go/examples/stringsplit/stringsplit.go | 1 +
.../windowed_wordcount/windowed_wordcount.go | 1 +
sdks/go/examples/wordcount/wordcount.go | 1 +
sdks/go/examples/xlang/sql/sql.go | 12 +-
sdks/go/examples/yatzy/yatzy.go | 1 +
sdks/go/pkg/beam/combine_test.go | 1 +
sdks/go/pkg/beam/core/runtime/xlangx/expand.go | 68 ++++
sdks/go/pkg/beam/core/runtime/xlangx/registry.go | 20 ++
.../pkg/beam/core/runtime/xlangx/registry_test.go | 12 +
sdks/go/pkg/beam/core/runtime/xlangx/resolve.go | 3 +
sdks/go/pkg/beam/io/xlang/jdbcio/jdbc.go | 114 ++++---
sdks/go/pkg/beam/partition_test.go | 1 +
.../beam/runners/dataflow/dataflowlib/execute.go | 9 +-
.../pkg/beam/runners/dataflow/dataflowlib/job.go | 29 +-
.../go/pkg/beam/transforms/filter/distinct_test.go | 1 +
sdks/go/pkg/beam/transforms/filter/filter_test.go | 1 +
sdks/go/pkg/beam/transforms/sql/sql.go | 3 +
sdks/go/pkg/beam/transforms/sql/sqlx/sqlx.go | 11 +-
sdks/go/prepare_go_version.sh | 19 +-
sdks/go/test/build.gradle | 3 +
sdks/go/test/integration/flags.go | 4 +
sdks/go/test/integration/integration.go | 8 +
sdks/go/test/integration/io/xlang/jdbc/jdbc.go | 6 +-
.../go/test/regression/coders/fromyaml/fromyaml.go | 4 +
sdks/go/test/run_validatesrunner_tests.sh | 41 ++-
.../container/license_scripts/dep_urls_java.yaml | 7 +-
.../container/license_scripts/license_script.sh | 3 +-
.../license_scripts/manual_licenses/xz/COPYING | 8 +
.../license_scripts/pull_licenses_java.py | 9 +-
.../beam/sdk/options/PipelineOptionsFactory.java | 28 +-
.../apache/beam/sdk/testing/TestStreamTest.java | 1 +
.../apache/beam/sdk/transforms/CombineFnsTest.java | 1 +
.../org/apache/beam/sdk/transforms/FilterTest.java | 1 +
sdks/java/extensions/sql/build.gradle | 2 +
sdks/java/extensions/sql/datacatalog/build.gradle | 2 +
sdks/java/harness/build.gradle | 1 +
.../apache/beam/sdk/io/aws2/kinesis/KinesisIO.java | 15 +-
.../beam/sdk/io/aws2/kinesis/KinesisReader.java | 71 ++--
.../beam/sdk/io/aws2/kinesis/KinesisSource.java | 100 +-----
.../beam/sdk/io/aws2/kinesis/ShardReadersPool.java | 23 +-
.../sdk/io/aws2/kinesis/KinesisReaderTest.java | 22 +-
.../sdk/io/aws2/kinesis/ShardReadersPoolTest.java | 73 ++--
.../io/gcp/bigquery/BigQueryStorageSourceBase.java | 19 +-
sdks/java/io/hadoop-common/build.gradle | 2 +
sdks/java/io/hadoop-format/build.gradle | 3 +
sdks/java/io/hcatalog/build.gradle | 1 +
.../apache_beam/coders/standard_coders_test.py | 14 +
sdks/python/apache_beam/dataframe/frames.py | 117 +++++--
.../apache_beam/dataframe/transforms_test.py | 2 +-
.../examples/cookbook/combiners_test.py | 1 +
.../apache_beam/examples/cookbook/filters_test.py | 1 +
.../examples/cookbook/multiple_output_pardo.py | 1 +
.../examples/streaming_wordcount_it_test.py | 1 +
sdks/python/apache_beam/examples/wordcount.py | 1 +
.../apache_beam/examples/wordcount_debugging.py | 1 +
.../apache_beam/examples/wordcount_minimal.py | 1 +
.../apache_beam/examples/wordcount_minimal_test.py | 1 +
sdks/python/apache_beam/examples/wordcount_test.py | 1 +
.../apache_beam/examples/wordcount_with_metrics.py | 1 +
sdks/python/apache_beam/metrics/metric_test.py | 1 +
...low_exercise_streaming_metrics_pipeline_test.py | 1 +
.../runners/portability/fn_api_runner/fn_runner.py | 7 +-
sdks/python/apache_beam/transforms/sql_test.py | 1 +
.../container/license_scripts/dep_urls_py.yaml | 2 +
.../container/py36/base_image_requirements.txt | 40 +--
.../container/py37/base_image_requirements.txt | 60 ++--
.../container/py38/base_image_requirements.txt | 60 ++--
.../documentation/resources/learning-resources.md | 10 +
.../content/en/get-started/try-beam-playground.md | 12 +-
.../partials/section-menu/en/get-started.html | 1 +
157 files changed, 1197 insertions(+), 657 deletions(-)
rename runners/{core-construction-java/src/test/java/org/apache/beam/runners/core/construction => java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire}/CommonCoderTest.java (86%)
create mode 100644 sdks/java/container/license_scripts/manual_licenses/xz/COPYING