You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2022/07/16 04:09:46 UTC
[beam] branch nightly-refs/heads/master updated (2b42751cb79 -> 4da1368a5bb)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 2b42751cb79 Merge pull request #22175: [BEAM-22089] Fix query retry in Java FirestoreIO
add 87267d57245 [BEAM-14101] Add Spark Receiver IO package and ReceiverBuilder (squashed)
add b0225ca084c Merge pull request #17111: [BEAM-14101] [CdapIO] Add ReceiverBuilder for SparkReceiverIO
add 2cb2ee2ba3b Moves timestamp skew override to correct place
add 0140c9c0020 Adds TestStream to verify window preservation of ElasticsearchIO#write
add 60e9573ef6f Removes unnecessary line
add 30d25de001e Adds validation that ES#Write outputs are in expected windows
add 132d634ce14 Updates window verification test to assert the exact docs in the window
add b9dabebc112 Uses guava Iterables over shaded avro version
add 673a4cc7930 Merge pull request #22183 from egalpin/egalpin/timestamp-skew-es
add 2f13855aff6 Merge pull request #22140 from [Playground Task] Sharing any code API
add b3963b75d35 Enables UnboundedSource wrapped SDF Kafka source by default for x-lang over the pure SDF source
add 896bf1af778 Merge pull request #22286: Enables UnboundedSource wrapped SDF Kafka source by default for Python x-lang
add c092af2b1ce RunInference documentation updates. (#22236)
add 72bf1a2df17 Don't try to parse non-flags as retained pipeline options.
add 3dcdcf3502b Merge pull request #22276 Don't try to parse non-flags as retained pipeline options.
add 7dcc6d8e612 Turn pr bot on for remaining common labels (#22257)
add 56ee83e7fd9 Reviewing the RunInference ReadMe file for clarity. (#22069)
add a165e4036e8 Bump protobufjs from 6.11.2 to 6.11.3 in /sdks/typescript
add 9bb5d5c7f36 Merge pull request #22071 Bump protobufjs from 6.11.2 to 6.11.3 in /sdks/typescript
add 23a85351f82 Collect heap profile on OOM on Dataflow (#22225)
add 2ba53174cf5 fixing the missing wrap around ring range read (#21786)
add 62a0d0a484f [Website] add playground section, update playground, update get-started #22301
add 29b26be0c2e Merge pull request #21959 from [Website] add playground section
add fa028d3dad8 Update RunInference documentation (#22250)
add 4da1368a5bb Rewrote Java multi-language pipeline quickstart (#22263)
No new revisions were added by this update.
Summary of changes:
.github/REVIEWERS.yml | 21 +
.github/workflows/build_playground_backend.yml | 1 +
playground/api/v1/api.proto | 39 ++
playground/backend/build.gradle.kts | 44 +-
playground/backend/cmd/server/controller.go | 78 +++
playground/backend/cmd/server/controller_test.go | 330 +++++++--
playground/backend/cmd/server/server.go | 55 +-
playground/backend/go.mod | 2 +
playground/backend/go.sum | 5 +-
playground/backend/internal/api/v1/api.pb.go | 757 ++++++++++++++++-----
playground/backend/internal/api/v1/api_grpc.pb.go | 76 +++
.../backend/internal/db/datastore/datastore_db.go | 196 ++++++
.../internal/db/datastore/datastore_db_test.go | 428 ++++++++++++
.../backend/internal/{utils/common.go => db/db.go} | 36 +-
.../backend/internal/db/entity/id_meta.go | 16 +-
.../backend/internal/db/entity/schema.go | 15 +-
.../backend/internal/db/entity/sdk.go | 17 +-
playground/backend/internal/db/entity/snippet.go | 71 ++
.../backend/internal/db/entity/snippet_test.go | 69 ++
.../backend/internal/db/mapper/datastore_mapper.go | 67 ++
.../internal/db/mapper/datastore_mapper_test.go | 134 ++++
.../backend/internal/db/mapper/mapper.go | 19 +-
.../internal/db/schema/migration/migration_v001.go | 92 +++
.../db/schema/migration/migration_v001_test.go | 96 +++
playground/backend/internal/db/schema/version.go | 62 ++
.../backend/internal/environment/application.go | 41 +-
.../internal/environment/environment_service.go | 42 +-
.../environment/environment_service_test.go | 14 +-
playground/backend/internal/utils/common.go | 15 +
playground/backend/internal/utils/db_utils.go | 52 ++
.../internal/utils/{common.go => db_utils_test.go} | 50 +-
playground/backend/internal/utils/file_utils.go | 102 +++
.../backend/internal/utils/file_utils_test.go | 255 +++++++
.../backend/internal/utils/test_utils.go | 20 +-
.../backend/internal/utils/test_utils_test.go | 32 +-
playground/backend/start_datastore_emulator.sh | 2 +-
playground/frontend/lib/api/v1/api.pb.dart | 307 +++++++++
playground/frontend/lib/api/v1/api.pbgrpc.dart | 54 ++
playground/frontend/lib/api/v1/api.pbjson.dart | 56 ++
playground/infrastructure/proxy/allow_list.py | 4 +-
scripts/ci/pr-bot/processNewPrs.ts | 29 +-
scripts/ci/pr-bot/processPrUpdate.ts | 13 -
sdks/go/container/boot.go | 20 +-
.../beam/core/runtime/harness/diagnostics_hook.go | 58 ++
sdks/go/pkg/beam/core/runtime/harness/harness.go | 6 +
sdks/go/pkg/beam/core/runtime/harness/init/init.go | 16 +
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 2 +-
sdks/go/pkg/beam/util/diagnostics/diagnostics.go | 134 ++++
sdks/go/pkg/beam/util/harnessopts/heap_dump.go | 39 ++
.../go/pkg/beam/util/harnessopts/heap_dump_test.go | 50 ++
sdks/go/pkg/beam/util/syscallx/syscall_default.go | 4 +
sdks/go/pkg/beam/util/syscallx/syscall_linux.go | 25 +-
sdks/go/test/integration/integration.go | 10 +
.../go/test/integration/primitives/heap_dump.go | 45 +-
.../test/integration/primitives/heap_dump_test.go | 66 ++
sdks/go/test/run_validatesrunner_tests.sh | 5 +-
.../org/apache/beam/sdk/io/cassandra/ReadFn.java | 62 +-
.../beam/sdk/io/cassandra/CassandraIOTest.java | 55 ++
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 6 +
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 6 +
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 6 +
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 6 +
.../elasticsearch/ElasticsearchIOTestCommon.java | 86 +++
.../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 14 +-
sdks/java/io/{cdap => sparkreceiver}/OWNERS | 0
sdks/java/io/sparkreceiver/build.gradle | 44 ++
.../beam/sdk/io/sparkreceiver/ReceiverBuilder.java | 91 +++
.../sdk/io/sparkreceiver/WrappedSupervisor.java | 89 +++
.../beam/sdk/io/sparkreceiver/package-info.java | 24 +
.../sdk/io/sparkreceiver/ReceiverBuilderTest.java | 89 +++
.../apache_beam/examples/inference/README.md | 83 ++-
.../transforms/elementwise/runinference.py | 153 +++++
.../transforms/elementwise/runinference_test.py | 125 ++++
sdks/python/apache_beam/io/kafka.py | 12 +-
sdks/python/apache_beam/ml/inference/base.py | 52 +-
.../python/apache_beam/options/pipeline_options.py | 5 +-
.../apache_beam/options/pipeline_options_test.py | 5 -
sdks/python/apache_beam/transforms/external.py | 39 +-
sdks/typescript/package-lock.json | 105 ++-
sdks/typescript/package.json | 2 +-
settings.gradle.kts | 1 +
.../site/assets/js/fix-playground-nested-scroll.js | 12 +-
website/www/site/assets/scss/_playground.sass | 10 +-
.../sdks/java-multi-language-pipelines.md | 331 ++++-----
.../documentation/sdks/python-machine-learning.md | 201 ++++++
.../site/content/en/documentation/sdks/python.md | 8 +-
.../transforms/python/elementwise/runinference.md | 105 +++
.../en/documentation/transforms/python/overview.md | 1 +
.../content/en/get-started/try-beam-playground.md | 6 +-
website/www/site/layouts/index.html | 24 +
website/www/site/layouts/partials/head.html | 3 +
.../www/site/layouts/partials/head_homepage.html | 3 +
.../www/site/layouts/partials/hooks/body-end.html | 3 -
.../partials/section-menu/en/documentation.html | 1 +
.../layouts/partials/section-menu/en/sdks.html | 1 +
.../{playground.html => playground_iframe.html} | 17 +-
96 files changed, 5468 insertions(+), 712 deletions(-)
create mode 100644 playground/backend/internal/db/datastore/datastore_db.go
create mode 100644 playground/backend/internal/db/datastore/datastore_db_test.go
copy playground/backend/internal/{utils/common.go => db/db.go} (54%)
copy sdks/go/pkg/beam/util/syscallx/syscall_default.go => playground/backend/internal/db/entity/id_meta.go (70%)
copy sdks/go/pkg/beam/util/syscallx/syscall_default.go => playground/backend/internal/db/entity/schema.go (70%)
copy sdks/go/pkg/beam/util/syscallx/syscall_default.go => playground/backend/internal/db/entity/sdk.go (64%)
create mode 100644 playground/backend/internal/db/entity/snippet.go
create mode 100644 playground/backend/internal/db/entity/snippet_test.go
create mode 100644 playground/backend/internal/db/mapper/datastore_mapper.go
create mode 100644 playground/backend/internal/db/mapper/datastore_mapper_test.go
copy sdks/go/pkg/beam/util/syscallx/syscall_default.go => playground/backend/internal/db/mapper/mapper.go (71%)
create mode 100644 playground/backend/internal/db/schema/migration/migration_v001.go
create mode 100644 playground/backend/internal/db/schema/migration/migration_v001_test.go
create mode 100644 playground/backend/internal/db/schema/version.go
create mode 100644 playground/backend/internal/utils/db_utils.go
copy playground/backend/internal/utils/{common.go => db_utils_test.go} (54%)
create mode 100644 playground/backend/internal/utils/file_utils.go
create mode 100644 playground/backend/internal/utils/file_utils_test.go
copy sdks/go/pkg/beam/util/syscallx/syscall_default.go => playground/backend/internal/utils/test_utils.go (71%)
copy sdks/go/pkg/beam/util/syscallx/syscall_default.go => playground/backend/internal/utils/test_utils_test.go (60%)
create mode 100644 sdks/go/pkg/beam/core/runtime/harness/diagnostics_hook.go
create mode 100644 sdks/go/pkg/beam/util/diagnostics/diagnostics.go
create mode 100644 sdks/go/pkg/beam/util/harnessopts/heap_dump.go
create mode 100644 sdks/go/pkg/beam/util/harnessopts/heap_dump_test.go
copy playground/backend/internal/utils/common.go => sdks/go/test/integration/primitives/heap_dump.go (56%)
create mode 100644 sdks/go/test/integration/primitives/heap_dump_test.go
copy sdks/java/io/{cdap => sparkreceiver}/OWNERS (100%)
create mode 100644 sdks/java/io/sparkreceiver/build.gradle
create mode 100644 sdks/java/io/sparkreceiver/src/main/java/org/apache/beam/sdk/io/sparkreceiver/ReceiverBuilder.java
create mode 100644 sdks/java/io/sparkreceiver/src/main/java/org/apache/beam/sdk/io/sparkreceiver/WrappedSupervisor.java
create mode 100644 sdks/java/io/sparkreceiver/src/main/java/org/apache/beam/sdk/io/sparkreceiver/package-info.java
create mode 100644 sdks/java/io/sparkreceiver/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ReceiverBuilderTest.java
create mode 100644 sdks/python/apache_beam/examples/snippets/transforms/elementwise/runinference.py
create mode 100644 sdks/python/apache_beam/examples/snippets/transforms/elementwise/runinference_test.py
create mode 100644 website/www/site/content/en/documentation/sdks/python-machine-learning.md
create mode 100644 website/www/site/content/en/documentation/transforms/python/elementwise/runinference.md
rename website/www/site/layouts/shortcodes/{playground.html => playground_iframe.html} (58%)