You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2022/10/28 06:01:07 UTC
[beam] branch nightly-refs/heads/master updated (ead2d2e8025 -> feaa1a277b2)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from ead2d2e8025 Support keyed executors in Samza Runner to process bundles for stateful ParDo (#23434)
add 5c0ff090d62 Fixing Get Started header link (#23490)
add 77f9a1749c1 Remove obsolete code from Spark 3 runner.
add 6809951b377 Merge pull request #23844: Remove obsolete code from Spark 3 runner.
add 6e395412ed4 Bump cloud.google.com/go/bigquery from 1.42.0 to 1.43.0 in /sdks (#23820)
add 37f9827f4f6 Bump google.golang.org/api from 0.100.0 to 0.101.0 in /sdks (#23842)
add 3038520522b Update ReadDataFromKinesis URN to registered URN (fixes #23693) (#23849)
add 932f87e7332 Bump dropwizard metrics-core for Spark 3 runner to match the version used in Spark 3.1 (addresses #23728)
add c670dfb081b Merge pull request #23802: Bump dropwizard metrics-core for Spark 3 runner to match the version used in Spark 3.1
add 840d66a1502 [23832] Remove ParquetIO.withSplit (closes #23832) (#23833)
add 6add48270af Bump github.com/spf13/cobra from 1.6.0 to 1.6.1 in /sdks (#23822)
add 8fb29f02efb [Go SDK] Add tests to the metrics package (#23769)
add d8966fd3b87 Bump cloud.google.com/go/pubsub from 1.25.1 to 1.26.0 in /sdks (#23823)
add fe791f61e5c Updated documentation to point to notebooks instead of having samples (#23858)
add 63b5f339115 Post 2.42.0 Updates to release guide (#23672)
add 9c442416023 Add Go usage instructions to download page. (#23698)
add 8808598b458 Deactivate Dask Runner code coverage tests as workaround (#23841)
add acd9cbeebc8 Use Akvelon editor (#23415) (#23825)
add 7ac0441485a [Tour Of Beam] User authorization part 1 (#23782)
add 131765baeaf [BEAM-23815] Fix Neo4j tests. (#23862)
add e435c91ae8d Add `arrow_type_compatibility` with `pyarrow.Table` to Beam Row `BatchConverter` (#23455)
add e35ddede9b2 [BEAM-23836] Updating documentation for cross-language Java pipelines using Python external transforms
add 80c86fa3bde update
add 1d5fd932a6a update
add 97985d1cded update
add 7f4c3d7b7de Merge pull request #23837 from ihji/BEAM-23836
add 09ae6810829 Reduce log spam of Py37PostCommit (#23829)
add a9e43e29efb Actually use the DatsetService that will be auto-closed (#23873)
add 29ff4be1d20 Migrate BINARY, VARBINARY, CHAR, VARCHAR jdbc logical types to portable (#23548)
add a62fe8f4eae [BEAM-12164] Feat: Added SpannerChangeStreamIT to Cloud Spanner Change Streams connector to test transaction tags filtering in the Change Stream records (#23284)
add feaa1a277b2 Use --release 8 for builds targeting Java 8 (#23771)
No new revisions were added by this update.
Summary of changes:
.../workflows/tour_of_beam_backend_integration.yml | 36 +-
CHANGES.md | 3 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 9 +
learning/tour-of-beam/backend/README.md | 72 ++--
learning/tour-of-beam/backend/auth.go | 92 +++++
learning/tour-of-beam/backend/docker-compose.yml | 11 +-
learning/tour-of-beam/backend/function.go | 83 ++++-
learning/tour-of-beam/backend/go.mod | 2 +
learning/tour-of-beam/backend/go.sum | 150 +++++++-
.../tour-of-beam/backend/integration_tests/api.go | 9 +
.../backend/integration_tests/auth_test.go | 186 ++++++++++
.../backend/integration_tests/client.go | 32 +-
.../emulators/datastore}/Dockerfile | 0
.../emulators/datastore}/start-datastore.sh | 0
.../emulators/firebase/Dockerfile | 18 +-
.../emulators/firebase/firebase.json | 12 +
.../backend/integration_tests/function_test.go | 28 +-
.../backend/integration_tests/local.sh | 16 +-
learning/tour-of-beam/backend/internal/entity.go | 16 +
.../backend/internal/service/content.go | 36 +-
.../backend/internal/storage/adapter.go | 11 +
.../backend/internal/storage/datastore.go | 57 +++
.../tour-of-beam/backend/internal/storage/iface.go | 4 +
.../backend/internal/{service => storage}/mock.go | 35 +-
.../backend/internal/storage/schema.go | 18 +
learning/tour-of-beam/backend/middleware.go | 19 +-
.../backend/samples/api/get_user_progress.json | 9 +
learning/tour-of-beam/frontend/pubspec.lock | 51 ++-
learning/tour-of-beam/frontend/pubspec.yaml | 4 -
.../beam/model/fnexecution/v1/standard_coders.yaml | 11 +
.../org/apache/beam/model/pipeline/v1/schema.proto | 28 ++
.../controllers/snippet_editing_controller.dart | 2 +-
.../playground_components/lib/src/theme/theme.dart | 4 +-
.../lib/src/widgets/editor_textarea.dart | 28 +-
.../frontend/playground_components/pubspec.yaml | 5 +-
playground/frontend/pubspec.lock | 60 +++-
.../utils/SerializationDebugger.java | 115 ------
.../structuredstreaming/utils/package-info.java | 20 --
runners/spark/spark_runner.gradle | 3 +-
sdks/go.mod | 14 +-
sdks/go.sum | 29 +-
sdks/go/pkg/beam/core/metrics/dumper_test.go | 49 +++
sdks/go/pkg/beam/core/metrics/metrics_test.go | 94 +++++
sdks/go/pkg/beam/core/metrics/store_test.go | 62 ++++
.../go/test/regression/coders/fromyaml/fromyaml.go | 1 +
sdks/java/container/agent/build.gradle | 10 +-
.../apache/beam/sdk/schemas/SchemaTranslation.java | 14 +-
.../beam/sdk/schemas/logicaltypes/FixedBytes.java | 69 ++--
.../beam/sdk/schemas/logicaltypes/FixedString.java | 77 ++++
.../sdk/schemas/logicaltypes/VariableBytes.java | 76 ++++
.../sdk/schemas/logicaltypes/VariableString.java | 76 ++++
.../apache/beam/sdk/schemas/utils/AvroUtils.java | 167 ++++-----
.../sdk/schemas/logicaltypes/LogicalTypesTest.java | 53 +++
.../gcp/bigquery/BigQueryStorageTableSource.java | 2 +-
.../changestreams/it/SpannerChangeStreamIT.java | 137 ++++++--
.../java/org/apache/beam/sdk/io/jdbc/JdbcUtil.java | 86 +++--
.../org/apache/beam/sdk/io/jdbc/LogicalTypes.java | 115 ++----
.../org/apache/beam/sdk/io/jdbc/SchemaUtil.java | 47 ++-
.../org/apache/beam/sdk/io/neo4j/Neo4jIOIT.java | 4 +-
.../org/apache/beam/sdk/io/parquet/ParquetIO.java | 236 +------------
.../apache/beam/sdk/io/parquet/ParquetIOTest.java | 109 +-----
.../apache/beam/sdk/tpcds/SqlTransformRunner.java | 1 -
sdks/python/apache_beam/coders/coder_impl.py | 7 +
.../io/external/xlang_jdbcio_it_test.py | 111 +++---
sdks/python/apache_beam/io/kinesis.py | 2 +-
sdks/python/apache_beam/portability/common_urns.py | 4 +
sdks/python/apache_beam/typehints/__init__.py | 7 +
.../typehints/arrow_batching_microbenchmark.py | 78 +++++
.../typehints/arrow_type_compatibility.py | 350 ++++++++++++++++++
...ch_test.py => arrow_type_compatibility_test.py} | 96 ++---
sdks/python/apache_beam/typehints/batch.py | 2 +
sdks/python/apache_beam/typehints/schemas.py | 389 ++++++++++++++++-----
sdks/python/apache_beam/typehints/schemas_test.py | 35 +-
sdks/python/test-suites/portable/common.gradle | 7 +
sdks/python/tox.ini | 2 +-
.../site/content/en/contribute/release-guide.md | 24 +-
.../content/en/documentation/programming-guide.md | 39 ++-
.../documentation/sdks/python-machine-learning.md | 54 +--
.../www/site/content/en/get-started/downloads.md | 8 +
website/www/site/layouts/partials/header.html | 6 +-
80 files changed, 2847 insertions(+), 1197 deletions(-)
create mode 100644 learning/tour-of-beam/backend/auth.go
create mode 100644 learning/tour-of-beam/backend/integration_tests/auth_test.go
rename learning/tour-of-beam/backend/{internal/storage/image => integration_tests/emulators/datastore}/Dockerfile (100%)
rename learning/tour-of-beam/backend/{internal/storage/image => integration_tests/emulators/datastore}/start-datastore.sh (100%)
copy website/www/site/i18n/home/calendar/en.yaml => learning/tour-of-beam/backend/integration_tests/emulators/firebase/Dockerfile (68%)
create mode 100644 learning/tour-of-beam/backend/integration_tests/emulators/firebase/firebase.json
rename learning/tour-of-beam/backend/internal/{service => storage}/mock.go (61%)
create mode 100644 learning/tour-of-beam/backend/samples/api/get_user_progress.json
delete mode 100644 runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/SerializationDebugger.java
delete mode 100644 runners/spark/3/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/package-info.java
create mode 100644 sdks/go/pkg/beam/core/metrics/dumper_test.go
create mode 100644 sdks/go/pkg/beam/core/metrics/store_test.go
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/FixedString.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/VariableBytes.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/VariableString.java
create mode 100644 sdks/python/apache_beam/typehints/arrow_batching_microbenchmark.py
create mode 100644 sdks/python/apache_beam/typehints/arrow_type_compatibility.py
copy sdks/python/apache_beam/typehints/{batch_test.py => arrow_type_compatibility_test.py} (64%)