You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2020/08/07 02:50:13 UTC
[beam] tag nightly-master updated (e0f02ea -> a46fbb8)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from e0f02ea (commit)
to a46fbb8 (commit)
from e0f02ea Migrate shared tag from tfx-bsl (#12468)
add 4107696 Move wordcount with metrics to its own file.
add 3dceb70 Cleanup WordCount example.
add 1e2b516 Merge pull request #12380 Cleanup WordCount example.
add 3558c96 Merge pull request #12149: [BEAM-9897] Add cross-language support to SnowflakeIO.Read
add 86b8326 Merge pull request #12151: [BEAM-9896] Add streaming for SnowflakeIO.Write to Java SDK
add 1f5aa5b [BEAM-7996] Add map & nil encoding to Go SDK.
add b1fff0a Merge pull request #12467 [BEAM-7996] Add map & nil encoding to Go SDK.
add 887d7e9 Update google-api-services* versions and google-client version to latest.
add 2ff9aae Set orderingKey field by method, not field name.
add 1e969b4 Merge pull request #12416: Update google-api-services versions.
add 01c7ad1 Simplify common patterns for pandas methods.
add 7bd1616 Use new infrastructure to simplify pandas implementation.
add 9d7f0c6 Merge pull request #12459 [BEAM-9547] Simplify pandas implementation.
add 18503a6 [BEAM-7996] Add support for MapType and Nulls in container types for Python RowCoder (#12426)
add 277832e [BEAM-10258] Support type hint annotations on PTransform's expand() (#12009)
add a76cf13 [BEAM-9421] Add Java snippets to NLP documentation.
add fdb3986 Merge pull request #12453 from mwalenia/BEAM-9421-nlp-transform-java-website-doc
add 4135968 [BEAM-10522] Added SnowflakeIO connector guide (#12296)
add 33b7b61 Add Additional Parameters lesson to Go SDK Katas
add 6077950 Edit lesson name and task description
add baf7978 Additional edits to task description
add cab5556 Update stepik course
add 064e6ba Merge pull request #12448 [BEAM-9679] Add a windowed PCollection lesson to Go SDK Katas
add e3e2989 [BEAM-10240] Support ZetaSQL DATETIME functions in BeamSQL (#12348)
add ab1c53f [BEAM-10633] UdfImpl should be able to return java.util.List.
add aa6e64c Merge pull request #12461 from amaliujia/rw-support_array_as_return_type
add 06a19b4 [BEAM-9615] Add initial Schema to Go conversions.
add 242f1f0 [BEAM-9615] Improve error handling on schema conv
add 8d03dc6 Merge pull request #12471 [BEAM-9615] Add initial Schema to Go conversions.
add a8a1f5a [BEAM-10543] Add new parameters to Kafka Read cross language configuration
add 2daf84d [BEAM-10543] Add new parameters to python wrapper of Kafka Read
add cf5c7ec [BEAM-10543] Modify Kafka python cross-language integration test to use new parameters
add fbf6e0a [BEAM-10543] Run Kafka cross-language integration test in python postcommit suite instead of separate task
add 12dc669 Merge pull request #12325: [BEAM-10543] Upgrade Kafka cross-language python tests
add bf62092 [BEAM-10135][BEAM-10136] Refactor jdbc external transform registrar
add 873237b Merge pull request #12423: [BEAM-10135][BEAM-10136] Refactor jdbc external transform registrar
add 94f9e7f Updating changes.md (#12424)
add 1671ce5 [BEAM-10645] Create context for allowing non-parallel dataframe operations. (#12476)
add 0b6af20 [BEAM-7390] Add sum code snippets
add a46fbb8 Merge pull request #12472 from davidcavazos/sum-docs
No new revisions were added by this update.
Summary of changes:
CHANGES.md | 3 +
build.gradle | 2 -
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 16 +-
examples/java/build.gradle | 3 +
.../apache/beam/examples/snippets/Snippets.java | 174 ++++++
.../katas/go/core_transforms/section-info.yaml | 1 +
.../lesson-info.yaml} | 11 +-
.../windowing/lesson-remote-info.yaml | 3 +
.../windowing/windowing/cmd/main.go | 48 ++
.../windowing/windowing/pkg/common/input.go | 42 ++
.../windowing/windowing/pkg/task/task.go | 54 ++
.../windowing/task-info.yaml} | 34 +-
.../windowing/windowing/task-remote-info.yaml | 2 +
.../go/core_transforms/windowing/windowing/task.md | 77 +++
.../windowing/windowing/test/task_test.go | 72 +++
learning/katas/go/course-remote-info.yaml | 2 +-
.../beam/model/fnexecution/v1/standard_coders.yaml | 30 +-
.../pipeline/src/main/proto/beam_runner_api.proto | 13 +-
model/pipeline/src/main/proto/schema.proto | 3 +
.../runners/core/construction/CommonCoderTest.java | 24 +-
sdks/go/pkg/beam/core/graph/coder/map.go | 102 ++++
sdks/go/pkg/beam/core/graph/coder/map_test.go | 98 ++++
sdks/go/pkg/beam/core/graph/coder/row.go | 40 +-
sdks/go/pkg/beam/core/graph/coder/row_test.go | 20 +-
.../pkg/beam/core/runtime/graphx/schema/schema.go | 269 ++++++++++
.../beam/core/runtime/graphx/schema/schema_test.go | 182 +++++++
.../beam/sdk/schemas/logicaltypes/DateTime.java | 83 +++
.../beam/sdk/schemas/logicaltypes/SqlTypes.java | 5 +
.../sql/meta/provider/datacatalog/SchemaUtils.java | 2 +-
.../extensions/sql/impl/ScalarFunctionImpl.java | 2 +-
.../sql/impl/planner/BeamRelDataTypeSystem.java | 2 +
.../sdk/extensions/sql/impl/rel/BeamCalcRel.java | 31 +-
.../extensions/sql/impl/utils/CalciteUtils.java | 39 +-
.../provider/bigquery/BeamSqlUnparseContext.java | 24 +-
.../sdk/extensions/sql/BeamComplexTypeTest.java | 162 +++++-
.../sdk/extensions/sql/BeamSqlDslUdfUdafTest.java | 39 ++
.../sql/impl/schema/BeamSqlRowCoderTest.java | 3 +
.../sdk/extensions/sql/zetasql/DateTimeUtils.java | 23 +-
.../zetasql/SupportedZetaSqlBuiltinFunctions.java | 36 +-
.../sql/zetasql/ZetaSqlBeamTranslationUtils.java | 25 +-
.../zetasql/ZetaSqlCalciteTranslationUtils.java | 6 +-
.../zetasql/translation/ExpressionConverter.java | 29 +-
.../zetasql/translation/TableScanConverter.java | 19 -
.../beam/sdk/extensions/sql/zetasql/TestInput.java | 29 +-
.../zetasql/ZetaSqlBeamTranslationUtilsTest.java | 14 +-
.../sql/zetasql/ZetaSqlDialectSpecTest.java | 2 +-
.../extensions/sql/zetasql/ZetaSqlTestBase.java | 1 +
.../sql/zetasql/ZetaSqlTimeFunctionsTest.java | 514 +++++++++++++++++-
.../beam/sdk/io/gcp/pubsub/PubsubJsonClient.java | 11 +-
.../sdk/io/gcp/pubsub/PubsubJsonClientTest.java | 2 +-
.../sdk/io/jdbc/CrossLanguageConfiguration.java | 76 ---
.../beam/sdk/io/jdbc/JdbcReadRowsRegistrar.java | 84 ---
.../beam/sdk/io/jdbc/JdbcTransformsRegistrar.java | 165 ++++++
.../beam/sdk/io/jdbc/JdbcWriteRegistrar.java | 67 ---
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 23 +-
.../beam/sdk/io/kafka/KafkaIOExternalTest.java | 14 +
sdks/java/io/snowflake/build.gradle | 2 +
.../io/snowflake/expansion-service/build.gradle | 38 ++
.../apache/beam/sdk/io/snowflake/SnowflakeIO.java | 588 ++++++++++++++++++---
.../sdk/io/snowflake/SnowflakePipelineOptions.java | 9 +-
.../credentials/SnowflakeCredentialsFactory.java | 50 +-
.../io/snowflake/crosslanguage/Configuration.java | 130 +++++
.../crosslanguage/SnowflakeReadRegistrar.java | 90 ++++
.../package-info.java} | 18 +-
.../StreamingLogLevel.java} | 7 +-
.../sdk/io/snowflake/services/ServiceConfig.java | 4 +
...onfig.java => SnowflakeBatchServiceConfig.java} | 67 ++-
...iceImpl.java => SnowflakeBatchServiceImpl.java} | 64 ++-
.../io/snowflake/services/SnowflakeService.java | 2 -
.../services/SnowflakeStreamingServiceConfig.java | 71 +++
.../services/SnowflakeStreamingServiceImpl.java | 72 +++
...mpl.java => FakeSnowflakeBatchServiceImpl.java} | 13 +-
.../test/FakeSnowflakeIngestManager.java} | 20 +-
.../test/FakeSnowflakeStreamingServiceImpl.java | 73 +++
.../beam/sdk/io/snowflake/test/TestUtils.java | 4 +
.../test/unit/read/SnowflakeIOReadTest.java | 18 +-
.../test/unit/write/CreateDispositionTest.java | 4 +-
.../unit/write/QueryDispositionLocationTest.java | 6 +-
.../test/unit/write/SchemaDispositionTest.java | 4 +-
.../test/unit/write/SnowflakeIOWriteTest.java | 70 ++-
.../test/unit/write/StreamingWriteTest.java | 321 +++++++++++
sdks/python/apache_beam/coders/coder_impl.py | 86 +++
sdks/python/apache_beam/coders/coders.py | 53 ++
.../apache_beam/coders/coders_test_common.py | 14 +-
sdks/python/apache_beam/coders/row_coder.py | 68 ++-
sdks/python/apache_beam/coders/row_coder_test.py | 19 +-
.../apache_beam/coders/standard_coders_test.py | 13 +-
sdks/python/apache_beam/dataframe/__init__.py | 4 +
sdks/python/apache_beam/dataframe/doctests.py | 4 +-
sdks/python/apache_beam/dataframe/expressions.py | 27 +
sdks/python/apache_beam/dataframe/frame_base.py | 79 ++-
.../apache_beam/dataframe/frame_base_test.py | 53 ++
sdks/python/apache_beam/dataframe/frames.py | 113 ++--
sdks/python/apache_beam/dataframe/frames_test.py | 31 ++
.../apache_beam/dataframe/transforms_test.py | 29 +-
.../examples/snippets/snippets_test_py3.py | 12 +
sdks/python/apache_beam/examples/wordcount.py | 91 +---
.../{wordcount.py => wordcount_with_metrics.py} | 0
sdks/python/apache_beam/io/external/snowflake.py | 204 +++++++
.../io/external/xlang_kafkaio_it_test.py | 56 +-
sdks/python/apache_beam/io/kafka.py | 50 +-
sdks/python/apache_beam/transforms/ptransform.py | 13 +
sdks/python/apache_beam/typehints/decorators.py | 70 +++
sdks/python/apache_beam/typehints/schemas.py | 5 +
.../typehints/typed_pipeline_test_py3.py | 130 +++++
.../apache_beam/typehints/typehints_test_py3.py | 220 +++++++-
sdks/python/test-suites/portable/common.gradle | 36 +-
settings.gradle | 1 +
.../en/documentation/io/built-in/snowflake.md | 364 +++++++++++++
.../en/documentation/patterns/ai-platform.md | 8 +-
.../en/documentation/sdks/python-type-safety.md | 13 +-
.../transforms/python/aggregation/sum.md | 58 +-
.../en/documentation/transforms/python/overview.md | 2 +-
.../content/en/roadmap/connectors-multi-sdk.md | 1 +
website/www/site/data/io_matrix.yaml | 1 +
.../partials/section-menu/en/documentation.html | 2 +
116 files changed, 5685 insertions(+), 847 deletions(-)
copy learning/katas/go/core_transforms/{section-info.yaml => windowing/lesson-info.yaml} (86%)
create mode 100644 learning/katas/go/core_transforms/windowing/lesson-remote-info.yaml
create mode 100644 learning/katas/go/core_transforms/windowing/windowing/cmd/main.go
create mode 100644 learning/katas/go/core_transforms/windowing/windowing/pkg/common/input.go
create mode 100644 learning/katas/go/core_transforms/windowing/windowing/pkg/task/task.go
copy learning/katas/go/core_transforms/{section-info.yaml => windowing/windowing/task-info.yaml} (64%)
create mode 100644 learning/katas/go/core_transforms/windowing/windowing/task-remote-info.yaml
create mode 100644 learning/katas/go/core_transforms/windowing/windowing/task.md
create mode 100644 learning/katas/go/core_transforms/windowing/windowing/test/task_test.go
create mode 100644 sdks/go/pkg/beam/core/graph/coder/map.go
create mode 100644 sdks/go/pkg/beam/core/graph/coder/map_test.go
create mode 100644 sdks/go/pkg/beam/core/runtime/graphx/schema/schema.go
create mode 100644 sdks/go/pkg/beam/core/runtime/graphx/schema/schema_test.go
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/DateTime.java
delete mode 100644 sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/CrossLanguageConfiguration.java
delete mode 100644 sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcReadRowsRegistrar.java
create mode 100644 sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcTransformsRegistrar.java
delete mode 100644 sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcWriteRegistrar.java
create mode 100644 sdks/java/io/snowflake/expansion-service/build.gradle
create mode 100644 sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/crosslanguage/Configuration.java
create mode 100644 sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/crosslanguage/SnowflakeReadRegistrar.java
copy sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/{services/SnowflakeService.java => crosslanguage/package-info.java} (68%)
copy sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/{services/ServiceConfig.java => enums/StreamingLogLevel.java} (89%)
rename sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/services/{SnowflakeServiceConfig.java => SnowflakeBatchServiceConfig.java} (60%)
rename sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/services/{SnowflakeServiceImpl.java => SnowflakeBatchServiceImpl.java} (79%)
create mode 100644 sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/services/SnowflakeStreamingServiceConfig.java
create mode 100644 sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/services/SnowflakeStreamingServiceImpl.java
rename sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/{FakeSnowflakeServiceImpl.java => FakeSnowflakeBatchServiceImpl.java} (89%)
copy sdks/java/io/snowflake/src/{main/java/org/apache/beam/sdk/io/snowflake/services/SnowflakeService.java => test/java/org/apache/beam/sdk/io/snowflake/test/FakeSnowflakeIngestManager.java} (66%)
create mode 100644 sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/FakeSnowflakeStreamingServiceImpl.java
create mode 100644 sdks/java/io/snowflake/src/test/java/org/apache/beam/sdk/io/snowflake/test/unit/write/StreamingWriteTest.java
copy sdks/python/apache_beam/examples/{wordcount.py => wordcount_with_metrics.py} (100%)
create mode 100644 sdks/python/apache_beam/io/external/snowflake.py
create mode 100644 website/www/site/content/en/documentation/io/built-in/snowflake.md