You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2020/10/14 02:47:06 UTC
[beam] tag nightly-master updated (f639a7e -> 7340bce)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from f639a7e (commit)
to 7340bce (commit)
from f639a7e Merge pull request #13079 Introduce intermediate superclass for both Dataframe and Series.
add e4c95f2 Add Relative Path Sensitivity (#12955)
add 11dc94d [BEAM-11056] Fix warning message and rename old APIs (#13080)
add 8f293a4 [BEAM-10969] Upgrade Spark to version 2.4.7
add c3477a8 Merge pull request #12937: [BEAM-10969] Upgrade Spark to version 2.4.7
add c9df735 Enable some supported ZetaSQL functions
add 93f7777 Merge pull request #13086: Enable some supported ZetaSQL functions
add 56313cc [BEAM-9309] Remove the READ urn/payload from the Java SDK harness.
add 0f7d423 Merge pull request #13074 from lukecwik/beam9309
add 1201986 [BEAM-10861]Adds PubSub Runner API encoding to Read/Write transforms (#12888)
add bbd407a [BEAM-8251] Add worker_region and worker_zone options (#13072)
add 919dc3c [BEAM-10720] Finish implementing StringMethods (cat, repeat) (#12705)
add e7e2d31 Add ability for HotKeyLogger to log a key.
add 7d150b0 Merge pull request #12984 from rohdesamuel/beam-10994-shared-2
add 42aff4d [BEAM-10009] Document micros_instant (#12921)
add 522b772 [BEAM-10987] Skip test_with_main_session on windows (#13099)
add 859ca46 [BEAM-9616] Allow new RegisterDoFn behaviour to be disabled (#13103)
add 3b2b07d [BEAM-11020] Adding multi-window splitting to Go SDF. (#13070)
add 0778277 [BEAM-10885] Add Avro support to Kafka table provider
add 986c21e Remove booleans, use List of values
add 56802e3 Fixes after CR
add 1a7462d Change payloadFormat to format like in text table provider
add e607bc6 Fix checkstyle
add 570eee6 Use SimpleFunction instead of SerializableFunction
add c2b8bf8 Remove getSchema() and listFrom(), make abstract generateRow, add generateEncodedPayload(i)
add af43533 Remove generification of KafkaTestRecord
add 14405f9 Add missing close paren
add 4f7cd00 Get rid of getTable() and KafkaTestTable inheritance
add 76b2e51 Fix comment
add 0b39204 Inline getRecordValueBytes
add 5f866ef Remove generification of generateProducerRecord
add 23894da Add full-stops to javadocs
add d121799 Move statistics tests to separate file
add 1f930ef4 Throw exception instead of returning null.
add 23c7188 Update create-external-table.md
add 1705c7b Update CHANGES.md
add 97ecb90 Use ByteArraySerializer as kafka value serializer
add d15b2f7 Update CHANGES.md
add ed18322 Update website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md
add 5d1d7a9 Update website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md
add d6a14e9 Update website/www/site/content/en/documentation/dsls/sql/extensions/create-external-table.md
add 7340bce [BEAM-10885] Add Avro support to Kafka table provider (#12827)
No new revisions were added by this update.
Summary of changes:
CHANGES.md | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 12 +-
.../pipeline/src/main/proto/beam_runner_api.proto | 54 +++-
.../worker/legacy-worker/build.gradle | 4 +-
.../beam/runners/dataflow/worker/HotKeyLogger.java | 31 +-
.../runners/dataflow/worker/HotKeyLoggerTest.java | 68 ++--
sdks/go/gogradle.lock | 16 +-
sdks/go/pkg/beam/core/runtime/exec/datasource.go | 24 +-
.../pkg/beam/core/runtime/exec/datasource_test.go | 4 +-
.../go/pkg/beam/core/runtime/exec/dynsplit_test.go | 13 +-
sdks/go/pkg/beam/core/runtime/exec/plan.go | 8 +-
sdks/go/pkg/beam/core/runtime/exec/sdf.go | 212 ++++++++++--
sdks/go/pkg/beam/core/runtime/exec/sdf_test.go | 359 +++++++++++++++++++--
sdks/go/pkg/beam/core/runtime/harness/harness.go | 28 +-
.../beam/io/rtrackers/offsetrange/offsetrange.go | 4 +-
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 6 +-
.../beam/runners/dataflow/dataflowlib/execute.go | 2 +-
.../pkg/beam/runners/dataflow/dataflowlib/job.go | 49 ++-
.../beam/runners/dataflow/dataflowlib/job_test.go | 126 ++++++++
sdks/go/pkg/beam/util/starcgenx/starcgenx.go | 23 ++
.../apache/beam/sdk/schemas/utils/AvroUtils.java | 63 ++++
.../beam/sdk/schemas/utils/AvroUtilsTest.java | 22 ++
.../meta/provider/kafka/BeamKafkaAvroTable.java | 89 +++++
.../sql/meta/provider/kafka/BeamKafkaCSVTable.java | 28 +-
.../sql/meta/provider/kafka/BeamKafkaTable.java | 4 +-
.../meta/provider/kafka/KafkaTableProvider.java | 21 +-
.../meta/provider/kafka/BeamKafkaCSVTableTest.java | 207 ------------
.../provider/kafka/BeamKafkaTableAvroTest.java | 93 ++++++
.../meta/provider/kafka/BeamKafkaTableCSVTest.java | 58 ++++
.../kafka/BeamKafkaTableStatisticsTest.java | 120 +++++++
.../meta/provider/kafka/BeamKafkaTableTest.java | 76 +++++
.../provider/kafka/KafkaTableProviderAvroIT.java} | 25 +-
...estRecord.java => KafkaTableProviderCSVIT.java} | 30 +-
...kaCSVTableIT.java => KafkaTableProviderIT.java} | 92 +++---
.../provider/kafka/KafkaTableProviderTest.java | 27 +-
.../sql/meta/provider/kafka/KafkaTestRecord.java | 8 +-
...{KafkaCSVTestTable.java => KafkaTestTable.java} | 28 +-
.../zetasql/SupportedZetaSqlBuiltinFunctions.java | 51 ++-
.../beam/fn/harness/BoundedSourceRunner.java | 184 -----------
.../beam/fn/harness/BoundedSourceRunnerTest.java | 211 ------------
sdks/java/maven-archetypes/examples/build.gradle | 12 +-
sdks/python/apache_beam/dataframe/frame_base.py | 15 +-
sdks/python/apache_beam/dataframe/frames.py | 75 ++++-
.../apache_beam/dataframe/pandas_doctests_test.py | 38 +--
.../apache_beam/dataframe/transforms_test.py | 21 ++
sdks/python/apache_beam/io/gcp/pubsub_test.py | 116 +++++++
sdks/python/apache_beam/io/iobase.py | 75 ++++-
.../runners/interactive/background_caching_job.py | 77 ++---
.../runners/interactive/interactive_beam.py | 145 +++++----
.../runners/interactive/interactive_beam_test.py | 2 +-
.../runners/interactive/interactive_runner.py | 2 +-
.../runners/interactive/interactive_runner_test.py | 2 +-
.../runners/interactive/options/capture_control.py | 4 +-
.../runners/interactive/pipeline_instrument.py | 18 +-
.../runners/interactive/recording_manager_test.py | 6 +-
.../apache_beam/runners/portability/stager_test.py | 4 +
sdks/python/build.gradle | 4 +-
.../dsls/sql/extensions/create-external-table.md | 12 +-
58 files changed, 2061 insertions(+), 1049 deletions(-)
create mode 100644 sdks/go/pkg/beam/runners/dataflow/dataflowlib/job_test.go
create mode 100644 sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaAvroTable.java
delete mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaCSVTableTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableAvroTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableCSVTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableStatisticsTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/BeamKafkaTableTest.java
copy sdks/java/{core/src/main/java/org/apache/beam/sdk/schemas/logicaltypes/NanosDuration.java => extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/KafkaTableProviderAvroIT.java} (57%)
copy sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/{KafkaTestRecord.java => KafkaTableProviderCSVIT.java} (57%)
rename sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/{KafkaCSVTableIT.java => KafkaTableProviderIT.java} (79%)
rename sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/kafka/{KafkaCSVTestTable.java => KafkaTestTable.java} (88%)
delete mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/BoundedSourceRunner.java
delete mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/BoundedSourceRunnerTest.java