You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2022/12/06 04:33:12 UTC
[beam] branch nightly-refs/heads/master updated (d15913b2891 -> ad92087babb)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from d15913b2891 Bump github.com/aws/aws-sdk-go-v2/feature/s3/manager in /sdks (#24501)
add a9dcf95453d [CdapIO] Add readme for CdapIO. Update readme for SparkReceiverIO. (#23959)
add c197579e9f2 [Spark Dataset runner] Add @Experimental and reduce visibility where possible.
add 66db2d81811 Merge pull request #24522: [Spark Dataset runner] Add @Experimental and reduce visibility where possible
add 54f579e45ab Fix grafana dashboard id (#24524)
add dd8c035937a [Spark runner] Support running (VR) tests with Java 17 (closes #24400) (#24401)
add 12d0ba99d19 Replaced deprecated finalize with DoFn Teardown (#24516)
add 719e5946420 Bump cloud.google.com/go/storage from 1.28.0 to 1.28.1 in /sdks (#24517)
add a88956b1e7c add clarifier to error message (#24449)
add 4a45066e653 Batch rename requests in fileio.WriteToFiles (#24341)
add 7701db80691 Bump golang.org/x/text from 0.4.0 to 0.5.0 in /sdks (#24520)
add 7e6e13e5864 Update nbconvert requirement in /sdks/python
add 3e4051cd005 Merge pull request #23240 from apache/dependabot/pip/sdks/python/nbconvert-gte-6.2.0-and-lt-8
add dbb584997c7 Support for JsonSchema in Kafka Read Schema Transform (#24272)
add 33af390d3a4 Run go fmt over full go directory with go 1.19 (#24525)
add 9b3b403ae9c Cloudbuild+manualsetup+playground (#24144)
add 6ba427ea6a1 Bump golang.org/x/sys from 0.2.0 to 0.3.0 in /sdks (#24519)
add 9f2de2ab8ed Bump cloud.google.com/go/bigtable from 1.18.0 to 1.18.1 in /sdks (#24500)
add d793631cd9c Update from interface{} -> any for core packages (#24505)
add cb329e410e9 Implement FileWriteSchemaTransformConfiguration (#24479)
add 92a65cbcf91 Bump cloud.google.com/go/pubsub from 1.27.0 to 1.27.1 in /sdks (#24518)
add 60aba30e12d [Playground] Healthcheck was added (#24227)
add ad92087babb Update dataflow container version for Pandas upgrade (#24532)
No new revisions were added by this update.
Summary of changes:
.../Python_ML_RunInference_Benchmark_Tests.json | 2 +-
.../cloudbuild/cloudbuild_pg_infra.yaml | 45 ++
.../cloudbuild/cloudbuild_pg_to_gke.yaml | 48 ++
playground/infrastructure/cloudbuild/env_init.sh | 44 ++
.../autoscaling-frontend.yaml} | 30 +-
.../{values.yaml => templates/autoscaling-go.yaml} | 30 +-
.../autoscaling-java.yaml} | 30 +-
.../autoscaling-python.yaml} | 30 +-
.../autoscaling-router.yaml} | 30 +-
.../autoscaling-scio.yaml} | 30 +-
.../templates/deployment-frontend.yml | 19 +
.../helm-playground/templates/deployment-go.yml | 21 +
.../helm-playground/templates/deployment-java.yml | 21 +
.../templates/deployment-python.yml | 21 +
.../templates/deployment-router.yml | 21 +
.../helm-playground/templates/deployment-scio.yml | 21 +
.../infrastructure/helm-playground/values.yaml | 11 +-
playground/terraform/README.md | 1 +
playground/terraform/build.gradle.kts | 16 +-
.../cloudbuild-manual-setup/01.setup/iam.tf | 44 ++
.../cloudbuild-manual-setup/01.setup/provider.tf | 20 +
.../cloudbuild-manual-setup/01.setup/services.tf | 33 +
.../cloudbuild-manual-setup/01.setup/terraform.tf | 28 +
.../cloudbuild-manual-setup/01.setup/variables.tf | 27 +
.../02.builders/provider.tf | 20 +
.../02.builders/terraform.tf | 28 +
.../02.builders/triggers.tf | 81 +++
.../02.builders/variables.tf | 108 ++++
.../cloudbuild-manual-setup/README.md | 116 ++++
.../SparkStructuredStreamingPipelineOptions.java | 2 +
.../SparkStructuredStreamingRunner.java | 2 +
.../translation/batch/Aggregators.java | 8 +-
.../batch/CombineGloballyTranslatorBatch.java | 2 +-
.../batch/CombineGroupedValuesTranslatorBatch.java | 2 +-
.../batch/GroupByKeyTranslatorBatch.java | 4 +-
.../translation/batch/ImpulseTranslatorBatch.java | 3 +-
.../batch/ReshuffleTranslatorBatch.java | 4 +-
runners/spark/spark_runner.gradle | 30 +-
sdks/go.mod | 10 +-
sdks/go.sum | 20 +-
sdks/go/cmd/starcgen/starcgen.go | 15 +-
sdks/go/examples/cookbook/tornadoes/tornadoes.go | 4 +-
.../debugging_wordcount/debugging_wordcount.go | 8 +-
sdks/go/examples/fhirio/import/import.go | 33 +-
.../fhirio/read_write_pubsub/read_write_pubsub.go | 37 +-
sdks/go/examples/kafka/taxi.go | 62 +-
.../go/examples/large_wordcount/large_wordcount.go | 8 +-
.../minimal_wordcount/minimal_wordcount.go | 8 +-
sdks/go/examples/native_wordcap/wordcap.go | 7 +-
sdks/go/examples/streaming_wordcap/wordcap.go | 7 +-
sdks/go/examples/stringsplit/stringsplit.go | 6 +-
sdks/go/examples/xlang/bigquery/wordcount.go | 61 +-
sdks/go/examples/yatzy/yatzy.go | 2 +-
sdks/go/pkg/beam/core/funcx/fn.go | 6 +-
sdks/go/pkg/beam/core/funcx/fn_test.go | 2 +-
sdks/go/pkg/beam/core/funcx/output.go | 7 +-
sdks/go/pkg/beam/core/funcx/output_test.go | 2 +-
sdks/go/pkg/beam/core/funcx/sideinput.go | 4 +-
sdks/go/pkg/beam/core/funcx/sideinput_test.go | 6 +-
sdks/go/pkg/beam/core/funcx/signature.go | 10 +-
sdks/go/pkg/beam/core/funcx/signature_test.go | 2 +-
sdks/go/pkg/beam/core/graph/bind.go | 20 +-
sdks/go/pkg/beam/core/graph/bind_test.go | 4 +-
sdks/go/pkg/beam/core/graph/coder/coder.go | 12 +-
sdks/go/pkg/beam/core/graph/coder/coder_test.go | 6 +-
sdks/go/pkg/beam/core/graph/coder/iterable.go | 8 +-
sdks/go/pkg/beam/core/graph/coder/iterable_test.go | 2 +-
sdks/go/pkg/beam/core/graph/coder/map_test.go | 2 +-
sdks/go/pkg/beam/core/graph/coder/nil_test.go | 2 +-
sdks/go/pkg/beam/core/graph/coder/registry.go | 10 +-
sdks/go/pkg/beam/core/graph/coder/registry_test.go | 4 +-
sdks/go/pkg/beam/core/graph/coder/row.go | 6 +-
sdks/go/pkg/beam/core/graph/coder/row_decoder.go | 16 +-
sdks/go/pkg/beam/core/graph/coder/row_encoder.go | 16 +-
sdks/go/pkg/beam/core/graph/coder/row_test.go | 36 +-
.../pkg/beam/core/graph/coder/testutil/testutil.go | 6 +-
.../core/graph/coder/testutil/testutil_test.go | 28 +-
sdks/go/pkg/beam/core/graph/fn.go | 27 +-
sdks/go/pkg/beam/core/graph/fn_test.go | 32 +-
.../beam/core/graph/window/trigger/trigger_test.go | 2 +-
sdks/go/pkg/beam/core/graph/xlang.go | 6 +-
sdks/go/pkg/beam/core/metrics/dumper.go | 10 +-
sdks/go/pkg/beam/core/metrics/dumper_test.go | 2 +-
sdks/go/pkg/beam/core/metrics/metrics.go | 11 +-
sdks/go/pkg/beam/core/metrics/store_test.go | 4 +-
sdks/go/pkg/beam/core/runtime/coderx/float_test.go | 2 +-
.../go/pkg/beam/core/runtime/coderx/string_test.go | 2 +-
.../go/pkg/beam/core/runtime/coderx/varint_test.go | 4 +-
sdks/go/pkg/beam/core/runtime/exec/coder.go | 10 +-
sdks/go/pkg/beam/core/runtime/exec/combine.go | 12 +-
sdks/go/pkg/beam/core/runtime/exec/combine_test.go | 58 +-
.../pkg/beam/core/runtime/exec/datasource_test.go | 42 +-
sdks/go/pkg/beam/core/runtime/exec/decode.go | 10 +-
.../go/pkg/beam/core/runtime/exec/dynsplit_test.go | 6 +-
sdks/go/pkg/beam/core/runtime/exec/emit.go | 6 +-
sdks/go/pkg/beam/core/runtime/exec/encode.go | 10 +-
sdks/go/pkg/beam/core/runtime/exec/fn.go | 26 +-
sdks/go/pkg/beam/core/runtime/exec/fn_test.go | 88 +--
sdks/go/pkg/beam/core/runtime/exec/fullvalue.go | 16 +-
.../pkg/beam/core/runtime/exec/fullvalue_test.go | 36 +-
sdks/go/pkg/beam/core/runtime/exec/hash.go | 12 +-
sdks/go/pkg/beam/core/runtime/exec/hash_test.go | 14 +-
sdks/go/pkg/beam/core/runtime/exec/input.go | 18 +-
.../beam/core/runtime/exec/optimized/callers.go | 176 +++---
.../beam/core/runtime/exec/optimized/callers.tmpl | 8 +-
.../beam/core/runtime/exec/optimized/decoders.go | 704 ++++++++++-----------
.../beam/core/runtime/exec/optimized/decoders.tmpl | 32 +-
.../beam/core/runtime/exec/optimized/emitters.go | 4 +-
.../beam/core/runtime/exec/optimized/emitters.tmpl | 4 +-
.../beam/core/runtime/exec/optimized/encoders.go | 672 ++++++++++----------
.../beam/core/runtime/exec/optimized/encoders.tmpl | 32 +-
.../pkg/beam/core/runtime/exec/optimized/inputs.go | 4 +-
.../beam/core/runtime/exec/optimized/inputs.tmpl | 4 +-
sdks/go/pkg/beam/core/runtime/exec/pardo.go | 4 +-
sdks/go/pkg/beam/core/runtime/exec/pardo_test.go | 18 +-
.../pkg/beam/core/runtime/exec/pcollection_test.go | 2 +-
sdks/go/pkg/beam/core/runtime/exec/sdf.go | 166 ++---
sdks/go/pkg/beam/core/runtime/exec/sdf_invokers.go | 138 ++--
.../beam/core/runtime/exec/sdf_invokers_test.go | 19 +-
sdks/go/pkg/beam/core/runtime/exec/sdf_test.go | 4 +-
sdks/go/pkg/beam/core/runtime/exec/sideinput.go | 6 +-
sdks/go/pkg/beam/core/runtime/exec/unit_test.go | 2 +-
sdks/go/pkg/beam/core/runtime/exec/userstate.go | 42 +-
.../pkg/beam/core/runtime/exec/userstate_test.go | 6 +-
sdks/go/pkg/beam/core/runtime/genx/genx.go | 6 +-
sdks/go/pkg/beam/core/runtime/genx/genx_test.go | 2 +-
sdks/go/pkg/beam/core/runtime/graphx/serialize.go | 2 +-
.../pkg/beam/core/runtime/graphx/translate_test.go | 8 +-
sdks/go/pkg/beam/core/runtime/graphx/user_test.go | 2 +-
sdks/go/pkg/beam/core/runtime/graphx/xlang.go | 4 +-
sdks/go/pkg/beam/core/runtime/harness/harness.go | 4 +-
sdks/go/pkg/beam/core/runtime/harness/session.go | 2 +-
.../runtime/harness/statecache/statecache_test.go | 4 +-
sdks/go/pkg/beam/core/runtime/symbols.go | 6 +-
sdks/go/pkg/beam/core/runtime/xlangx/payload.go | 4 +-
sdks/go/pkg/beam/core/runtime/xlangx/registry.go | 11 +-
.../pkg/beam/core/runtime/xlangx/registry_test.go | 2 +-
sdks/go/pkg/beam/core/runtime/xlangx/resolve.go | 9 +-
sdks/go/pkg/beam/core/sdf/lock.go | 6 +-
sdks/go/pkg/beam/core/sdf/sdf.go | 6 +-
sdks/go/pkg/beam/core/state/state.go | 34 +-
sdks/go/pkg/beam/core/state/state_test.go | 108 ++--
sdks/go/pkg/beam/core/typex/class_test.go | 6 +-
sdks/go/pkg/beam/core/typex/special.go | 14 +-
sdks/go/pkg/beam/core/util/ioutilx/read.go | 1 +
sdks/go/pkg/beam/core/util/jsonx/jsonx.go | 8 +-
sdks/go/pkg/beam/core/util/reflectx/call.go | 18 +-
sdks/go/pkg/beam/core/util/reflectx/call_test.go | 16 +-
sdks/go/pkg/beam/core/util/reflectx/calls.go | 446 ++++++-------
sdks/go/pkg/beam/core/util/reflectx/calls.tmpl | 10 +-
sdks/go/pkg/beam/core/util/reflectx/functions.go | 4 +-
sdks/go/pkg/beam/core/util/reflectx/structs.go | 8 +-
.../go/pkg/beam/core/util/reflectx/structs_test.go | 4 +-
sdks/go/pkg/beam/core/util/reflectx/types_test.go | 12 +-
sdks/go/pkg/beam/core/util/reflectx/util.go | 4 +-
sdks/go/pkg/beam/core/util/reflectx/util_test.go | 4 +-
sdks/go/pkg/beam/core/util/symtab/symtab_test.go | 2 +-
sdks/go/pkg/beam/forward.go | 32 +-
sdks/go/pkg/beam/gbk.go | 4 +-
sdks/go/pkg/beam/internal/errors/errors.go | 12 +-
sdks/go/pkg/beam/io/databaseio/mapper.go | 10 +-
sdks/go/pkg/beam/io/databaseio/util.go | 2 +-
sdks/go/pkg/beam/io/datastoreio/datastore.go | 1 -
sdks/go/pkg/beam/io/parquetio/parquetio.go | 38 +-
sdks/go/pkg/beam/io/synthetic/source.go | 27 +-
sdks/go/pkg/beam/io/synthetic/step.go | 6 +-
sdks/go/pkg/beam/io/xlang/bigqueryio/bigquery.go | 71 ++-
sdks/go/pkg/beam/io/xlang/debeziumio/debezium.go | 32 +-
sdks/go/pkg/beam/io/xlang/jdbcio/jdbc.go | 71 ++-
sdks/go/pkg/beam/io/xlang/kafkaio/kafka.go | 31 +-
.../beam/model/fnexecution_v1/beam_fn_api.pb.go | 38 +-
.../beam/model/jobmanagement_v1/beam_job_api.pb.go | 11 +-
.../beam/model/pipeline_v1/beam_runner_api.pb.go | 42 +-
sdks/go/pkg/beam/pardo.go | 146 +++--
sdks/go/pkg/beam/partition.go | 4 +-
.../beam/runners/dataflow/dataflowlib/messages.go | 18 +-
.../beam/runners/universal/runnerlib/compile.go | 8 +-
sdks/go/pkg/beam/schema.go | 10 +-
sdks/go/pkg/beam/testing/ptest/ptest.go | 1 -
sdks/go/pkg/beam/transforms/filter/filter.go | 16 +-
sdks/go/pkg/beam/transforms/stats/max.go | 5 +-
sdks/go/pkg/beam/transforms/stats/mean.go | 5 +-
sdks/go/pkg/beam/transforms/stats/min.go | 5 +-
sdks/go/pkg/beam/transforms/stats/sum.go | 5 +-
sdks/go/pkg/beam/transforms/top/top.go | 10 +-
.../pkg/beam/transforms/xlang/python/external.go | 31 +-
sdks/go/pkg/beam/xlang.go | 61 +-
sdks/go/test/integration/expansions.go | 23 +-
sdks/go/test/integration/flags.go | 9 +-
.../go/test/integration/io/xlang/jdbc/jdbc_test.go | 2 +-
sdks/java/io/cdap/README.md | 145 +++++
sdks/java/io/cdap/build.gradle | 1 -
sdks/java/io/fileschematransform/OWNERS | 5 +
sdks/java/io/fileschematransform/build.gradle | 46 ++
.../FileWriteSchemaTransformConfiguration.java | 245 +++++++
.../FileWriteSchemaTransformFormatProvider.java | 37 ++
.../FileWriteSchemaTransformFormatProviders.java | 124 ++++
.../sdk/io/fileschematransform/package-info.java | 20 +
...ileWriteSchemaTransformFormatProvidersTest.java | 43 ++
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 4 +-
sdks/java/io/kafka/build.gradle | 3 +
... => KafkaReadSchemaTransformConfiguration.java} | 16 +-
....java => KafkaReadSchemaTransformProvider.java} | 30 +-
...a => KafkaReadSchemaTransformProviderTest.java} | 55 +-
sdks/java/io/sparkreceiver/2/README.md | 40 +-
sdks/java/io/sparkreceiver/2/build.gradle | 1 -
sdks/python/apache_beam/io/filebasedsink.py | 3 +-
sdks/python/apache_beam/io/fileio.py | 65 +-
sdks/python/apache_beam/io/gcp/gcsio.py | 5 +-
.../apache_beam/runners/dataflow/internal/names.py | 4 +-
sdks/python/setup.py | 2 +-
settings.gradle.kts | 1 +
212 files changed, 4032 insertions(+), 2478 deletions(-)
create mode 100644 playground/infrastructure/cloudbuild/cloudbuild_pg_infra.yaml
create mode 100644 playground/infrastructure/cloudbuild/cloudbuild_pg_to_gke.yaml
create mode 100644 playground/infrastructure/cloudbuild/env_init.sh
copy playground/infrastructure/helm-playground/{values.yaml => templates/autoscaling-frontend.yaml} (62%)
copy playground/infrastructure/helm-playground/{values.yaml => templates/autoscaling-go.yaml} (62%)
copy playground/infrastructure/helm-playground/{values.yaml => templates/autoscaling-java.yaml} (62%)
copy playground/infrastructure/helm-playground/{values.yaml => templates/autoscaling-python.yaml} (62%)
copy playground/infrastructure/helm-playground/{values.yaml => templates/autoscaling-router.yaml} (62%)
copy playground/infrastructure/helm-playground/{values.yaml => templates/autoscaling-scio.yaml} (62%)
create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/iam.tf
create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/provider.tf
create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/services.tf
create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/terraform.tf
create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/01.setup/variables.tf
create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/provider.tf
create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/terraform.tf
create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/triggers.tf
create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/02.builders/variables.tf
create mode 100644 playground/terraform/infrastructure/cloudbuild-manual-setup/README.md
create mode 100644 sdks/java/io/cdap/README.md
create mode 100644 sdks/java/io/fileschematransform/OWNERS
create mode 100644 sdks/java/io/fileschematransform/build.gradle
create mode 100644 sdks/java/io/fileschematransform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformConfiguration.java
create mode 100644 sdks/java/io/fileschematransform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvider.java
create mode 100644 sdks/java/io/fileschematransform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviders.java
create mode 100644 sdks/java/io/fileschematransform/src/main/java/org/apache/beam/sdk/io/fileschematransform/package-info.java
create mode 100644 sdks/java/io/fileschematransform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvidersTest.java
rename sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/{KafkaSchemaTransformReadConfiguration.java => KafkaReadSchemaTransformConfiguration.java} (88%)
rename sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/{KafkaSchemaTransformReadProvider.java => KafkaReadSchemaTransformProvider.java} (86%)
rename sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/{KafkaSchemaTransformReadProviderTest.java => KafkaReadSchemaTransformProviderTest.java} (66%)