You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2023/02/23 05:40:06 UTC
[beam] branch nightly-refs/heads/master updated (921bc7b9470 -> 011296c1465)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 921bc7b9470 [prism] Add in element manager (#25565)
add 903fbf9e721 Better error for missing timezone info (#25444)
add b92600adafe Revert "Replace more uses of `ClassLoadingStrategy.Default.INJECTION` (#23210)" (#25566)
add e80596f5d41 Use context param and error return value in mongodbio.Read SDF methods (#25536)
add af2a86139cf Revert workers for java tests (#25579)
add 3bfc3136a2a remove unnecessary error check (#25576)
add 3a6259900e6 Making the default BigQuery write disposition to be WRITE_APPEND (#25434)
add 33750c1dabf [Python] Support loading of TF models with saved weights (#25496)
add 5ace2ec4caf Bump github.com/aws/aws-sdk-go-v2/config in /sdks (#25572)
add 5b4edaf83bf [#24789][prism] V0 Go Direct Runner Replacement, Prism. (#25568)
add a6bd21a5616 Add retry logic to Python boot script.
add e99999509f8 More verbose messaging.
add 8cfee7d05e6 Merge pull request #25473 Add retry logic to Python boot script.
add d8e5047dad3 Bump github.com/testcontainers/testcontainers-go in /sdks (#25517)
add 95a6566107e [Avro] Use "extensions/avro" instead of avro from"core" in Java SDK modules (#24992)
add 83cfdb829b8 PCollection data sampling for Java SDK harness #25064 (#25354)
add 85817b6b343 Add partitioned reads to JDBC SchemaIO (#25577)
add 42827acbb64 Deprecate Avro API classes in "sdks/java/core"
add 5b59377ff61 Merge pull request #25534: [Avro] Deprecate Avro API classes in "sdks/java/core"
add cebe23b37e2 Bump github.com/aws/aws-sdk-go from 1.33.0 to 1.34.0 in /sdks (#25469)
add 50e5d728fc1 ML model evaluation website page (#25589)
add 2e7584cc43a Add Data Sampling capability URN to Java SDK (#25593)
add 93c1e92f371 Add anchor link for the logo wall (#25594)
add 4ce8eeda196 Read API Source v2 (#25392)
add 27c55107e56 Moving to 2.47.0-SNAPSHOT on master branch.
add 011296c1465 Apply shallow clone for jenkins (#25595)
No new revisions were added by this update.
Summary of changes:
.github/workflows/java_tests.yml | 8 +-
.test-infra/jenkins/CommonJobProperties.groovy | 4 +
CHANGES.md | 8 +-
build.gradle.kts | 1 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
examples/java/build.gradle | 1 +
.../beam/examples/complete/AutoComplete.java | 2 +-
.../beam/examples/complete/TrafficMaxLaneFlow.java | 2 +-
.../beam/examples/complete/TrafficRoutes.java | 2 +-
.../beam/examples/complete/game/UserScore.java | 2 +-
.../complete/kafkatopubsub/avro/AvroDataClass.java | 2 +-
.../kafkatopubsub/transforms/FormatTransform.java | 2 +-
.../apache/beam/examples/snippets/Snippets.java | 2 +-
.../transforms/io/gcp/bigquery/BigQueryMyData.java | 2 +-
.../examples/subprocess/utils/ExecutableFile.java | 2 +-
.../examples/complete/game/LeaderBoardTest.java | 2 +-
.../complete/game/StatefulTeamScoreTest.java | 2 +-
examples/kotlin/build.gradle | 1 +
.../beam/examples/kotlin/snippets/Snippets.kt | 2 +-
gradle.properties | 4 +-
runners/core-construction-java/build.gradle | 2 +
.../construction/AvroGenericCoderRegistrar.java | 2 +-
.../construction/AvroGenericCoderTranslator.java | 2 +-
.../runners/core/construction/Environments.java | 1 +
.../core/construction/CoderTranslationTest.java | 2 +-
.../core/construction/EnvironmentsTest.java | 3 +
runners/direct-java/build.gradle | 4 +-
runners/google-cloud-dataflow-java/build.gradle | 2 +
.../util/AvroCoderCloudObjectTranslator.java | 2 +-
.../runners/dataflow/util/CloudObjectsTest.java | 2 +-
.../google-cloud-dataflow-java/worker/build.gradle | 1 +
.../runners/dataflow/worker/AvroByteReader.java | 4 +-
.../translation/SparkSessionFactory.java | 16 +-
runners/spark/spark_runner.gradle | 2 +
.../beam/runners/spark/io/AvroPipelineTest.java | 2 +-
sdks/go.mod | 51 +-
sdks/go.sum | 743 +---------
sdks/go/pkg/beam/core/core.go | 2 +-
sdks/go/pkg/beam/core/runtime/xlangx/expand.go | 6 +-
sdks/go/pkg/beam/io/mongodbio/read.go | 29 +-
sdks/go/pkg/beam/runners/prism/README.md | 1 +
.../pkg/beam/runners/prism/internal/coders_test.go | 6 +-
sdks/go/pkg/beam/runners/prism/internal/execute.go | 278 +++-
.../beam/runners/prism/internal/execute_test.go | 417 ++++++
.../beam/runners/prism/internal/separate_test.go | 595 ++++++++
sdks/go/pkg/beam/runners/prism/internal/stage.go | 400 ++++++
sdks/go/pkg/beam/runners/prism/prism.go | 48 +
.../java/org/apache/beam/sdk/coders/AvroCoder.java | 5 +
.../apache/beam/sdk/coders/AvroGenericCoder.java | 10 +-
.../org/apache/beam/sdk/coders/DefaultCoder.java | 2 +-
.../apache/beam/sdk/coders/RowCoderGenerator.java | 7 +-
.../main/java/org/apache/beam/sdk/io/AvroIO.java | 6 +
.../apache/beam/sdk/io/AvroSchemaIOProvider.java | 6 +
.../main/java/org/apache/beam/sdk/io/AvroSink.java | 10 +-
.../java/org/apache/beam/sdk/io/AvroSource.java | 5 +
.../beam/sdk/io/ConstantAvroDestination.java | 10 +-
.../beam/sdk/io/DynamicAvroDestinations.java | 6 +
.../beam/sdk/io/SerializableAvroCodecFactory.java | 7 +
.../apache/beam/sdk/schemas/AvroRecordSchema.java | 6 +
.../org/apache/beam/sdk/schemas/io/Providers.java | 33 +-
.../io/payloads/AvroPayloadSerializerProvider.java | 8 +
.../beam/sdk/schemas/utils/AvroByteBuddyUtils.java | 13 +-
.../apache/beam/sdk/schemas/utils/AvroUtils.java | 6 +
.../beam/sdk/schemas/utils/ByteBuddyUtils.java | 4 +-
.../beam/sdk/schemas/utils/JavaBeanUtils.java | 11 +-
.../apache/beam/sdk/schemas/utils/POJOUtils.java | 10 +-
.../sdk/schemas/utils/SelectByteBuddyHelpers.java | 7 +-
.../reflect/ByteBuddyOnTimerInvokerFactory.java | 10 +-
.../org/apache/beam/sdk/util/ByteBuddyUtils.java | 57 -
.../org/apache/beam/sdk/io/TextIOWriteTest.java | 30 +-
.../org/apache/beam/sdk/transforms/LatestTest.java | 4 +-
sdks/java/extensions/avro/build.gradle | 4 +-
.../extensions/avro/coders/DefaultCoderTest.java | 61 +
.../extensions/avro/schemas/SchemaCoderTest.java | 196 +++
.../avro/schemas/transforms/ConvertTest.java | 147 ++
sdks/java/extensions/sketching/build.gradle | 1 +
.../sketching/ApproximateDistinctTest.java | 2 +-
.../sketching/SketchFrequenciesTest.java | 2 +-
sdks/java/extensions/sql/build.gradle | 1 +
.../sql/meta/provider/avro/AvroTableProvider.java | 4 +-
.../sql/meta/provider/parquet/ParquetTable.java | 2 +-
.../provider/kafka/BeamKafkaTableAvroTest.java | 4 +-
.../meta/provider/kafka/KafkaTableProviderIT.java | 2 +-
.../provider/pubsub/PubsubTableProviderIT.java | 4 +-
sdks/java/harness/build.gradle | 1 +
.../java/org/apache/beam/fn/harness/FnHarness.java | 14 +-
.../fn/harness/control/ProcessBundleHandler.java | 18 +-
.../harness/data/PCollectionConsumerRegistry.java | 59 +-
.../apache/beam/fn/harness/debug/DataSampler.java | 125 ++
.../beam/fn/harness/debug/OutputSampler.java | 129 ++
.../beam/fn/harness/debug/package-info.java} | 17 +-
.../harness/control/ProcessBundleHandlerTest.java | 48 +-
.../data/PCollectionConsumerRegistryTest.java | 60 +
.../beam/fn/harness/debug/DataSamplerTest.java | 273 ++++
.../beam/fn/harness/debug/OutputSamplerTest.java | 188 +++
sdks/java/io/amazon-web-services/build.gradle | 1 +
.../sdk/io/aws/dynamodb/DynamoDBIOWriteTest.java | 2 +-
sdks/java/io/amazon-web-services2/build.gradle | 1 +
.../sdk/io/aws2/dynamodb/DynamoDBIOWriteTest.java | 2 +-
sdks/java/io/cdap/build.gradle | 1 +
.../apache/beam/sdk/io/cdap/TestRowDBWritable.java | 2 +-
sdks/java/io/file-based-io-tests/build.gradle | 1 +
.../java/org/apache/beam/sdk/io/avro/AvroIOIT.java | 4 +-
.../apache/beam/sdk/io/parquet/ParquetIOIT.java | 2 +-
sdks/java/io/file-schema-transform/build.gradle | 1 +
.../AvroWriteSchemaTransformFormatProvider.java | 6 +-
.../FileWriteSchemaTransformFormatProviders.java | 2 +-
.../ParquetWriteSchemaTransformFormatProvider.java | 2 +-
...FileWriteSchemaTransformFormatProviderTest.java | 6 +-
...FileWriteSchemaTransformFormatProviderTest.java | 2 +-
sdks/java/io/google-cloud-platform/build.gradle | 2 +
.../AvroGenericRecordToStorageApiProto.java | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 361 +++--
.../beam/sdk/io/gcp/bigquery/BigQueryOptions.java | 11 +
.../sdk/io/gcp/bigquery/BigQueryQuerySource.java | 2 +-
.../io/gcp/bigquery/BigQueryQuerySourceDef.java | 2 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 2 +-
.../sdk/io/gcp/bigquery/BigQuerySourceDef.java | 2 +-
.../gcp/bigquery/BigQueryStorageArrowReader.java | 2 +-
.../io/gcp/bigquery/BigQueryStorageSourceBase.java | 67 +-
.../BigQueryStorageStreamBundleSource.java | 381 ++++++
.../sdk/io/gcp/bigquery/BigQueryTableSource.java | 2 +-
.../io/gcp/bigquery/BigQueryTableSourceDef.java | 2 +-
...StorageApiDynamicDestinationsGenericRecord.java | 2 +-
...ueryStorageWriteApiSchemaTransformProvider.java | 3 +-
.../beam/sdk/io/gcp/pubsub/PubsubClient.java | 2 +-
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 4 +-
.../PubsubLiteReadSchemaTransformProvider.java | 2 +-
.../PubsubLiteWriteSchemaTransformProvider.java | 2 +-
.../model/ChangeStreamRecordMetadata.java | 2 +-
.../changestreams/model/ChildPartition.java | 2 +-
.../changestreams/model/ChildPartitionsRecord.java | 2 +-
.../spanner/changestreams/model/ColumnType.java | 2 +-
.../changestreams/model/DataChangeRecord.java | 2 +-
.../changestreams/model/HeartbeatRecord.java | 2 +-
.../io/gcp/spanner/changestreams/model/Mod.java | 2 +-
.../gcp/spanner/changestreams/model/ModType.java | 2 +-
.../changestreams/model/PartitionMetadata.java | 2 +-
.../gcp/spanner/changestreams/model/TypeCode.java | 2 +-
.../changestreams/model/ValueCaptureType.java | 2 +-
.../AvroGenericRecordToStorageApiProtoTest.java | 6 +-
.../sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java | 2 +-
.../sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 2 +-
.../io/gcp/bigquery/BigQueryIOStorageReadTest.java | 2 +-
...ryIOStorageReadWithStreamBundleSourceTest.java} | 1418 +++++++++-----------
.../sdk/io/gcp/bigquery/BigQueryUtilsTest.java | 2 +-
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 2 +-
.../PubsubReadSchemaTransformProviderTest.java | 2 +-
.../sdk/io/gcp/pubsub/PubsubRowToMessageTest.java | 2 +-
...bsubSchemaTransformMessageToRowFactoryTest.java | 2 +-
.../PubsubWriteSchemaTransformProviderTest.java | 2 +-
.../encoder/TimestampEncodingTest.java | 2 +-
sdks/java/io/hadoop-format/build.gradle | 1 +
.../apache/beam/sdk/io/hadoop/format/Employee.java | 2 +-
.../io/hadoop/format/HadoopFormatIOReadTest.java | 2 +-
.../sdk/io/hadoop/format/TestRowDBWritable.java | 2 +-
sdks/java/io/influxdb/build.gradle | 1 +
.../org/apache/beam/sdk/io/influxdb/Model.java | 2 +-
sdks/java/io/jdbc/build.gradle | 1 +
.../beam/sdk/io/jdbc/JdbcSchemaIOProvider.java | 63 +-
.../beam/sdk/io/jdbc/JdbcSchemaIOProviderTest.java | 129 ++
.../apache/beam/sdk/io/jdbc/SchemaUtilTest.java | 2 +-
sdks/java/io/kafka/build.gradle | 2 +
...onfluentSchemaRegistryDeserializerProvider.java | 2 +-
.../beam/sdk/io/kafka/KafkaCheckpointMark.java | 2 +-
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 2 +-
.../io/kafka/KafkaReadSchemaTransformProvider.java | 4 +-
.../beam/sdk/io/kafka/KafkaUnboundedSource.java | 2 +-
.../kafka/KafkaWriteSchemaTransformProvider.java | 2 +-
...uentSchemaRegistryDeserializerProviderTest.java | 2 +-
.../org/apache/beam/sdk/io/kafka/KafkaIOIT.java | 2 +-
sdks/java/io/parquet/build.gradle | 2 +
.../org/apache/beam/sdk/io/parquet/ParquetIO.java | 4 +-
.../apache/beam/sdk/io/parquet/ParquetIOTest.java | 4 +-
sdks/java/io/snowflake/build.gradle | 1 +
.../test/unit/read/SnowflakeIOReadTest.java | 2 +-
sdks/java/testing/expansion-service/build.gradle | 1 +
.../testing/expansion/TestExpansionService.java | 2 +-
sdks/java/testing/nexmark/build.gradle | 1 +
.../apache/beam/sdk/nexmark/NexmarkLauncher.java | 2 +-
.../org/apache/beam/sdk/nexmark/NexmarkUtils.java | 2 +-
sdks/python/apache_beam/coders/coder_impl.pxd | 4 +
sdks/python/apache_beam/coders/coder_impl.py | 18 +-
sdks/python/apache_beam/coders/coders.py | 19 +
.../apache_beam/coders/coders_test_common.py | 1 +
sdks/python/apache_beam/coders/row_coder.py | 3 +
sdks/python/apache_beam/coders/slow_stream.py | 6 +
sdks/python/apache_beam/coders/stream.pxd | 3 +
sdks/python/apache_beam/coders/stream.pyx | 16 +
sdks/python/apache_beam/coders/stream_test.py | 9 +
.../apache_beam/examples/inference/README.md | 53 +
.../inference/tensorflow_mnist_with_weights.py | 93 ++
.../io/external/xlang_jdbcio_it_test.py | 18 +
sdks/python/apache_beam/io/jdbc.py | 37 +-
.../ml/inference/tensorflow_inference.py | 29 +
.../ml/inference/tensorflow_inference_it_test.py | 31 +
.../runners/direct/transform_evaluator.py | 5 +-
sdks/python/apache_beam/utils/timestamp.py | 5 +
sdks/python/apache_beam/utils/timestamp_test.py | 5 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/container/boot.go | 49 +-
sdks/python/tox.ini | 2 +-
.../en/documentation/ml/model-evaluation.md | 85 ++
website/www/site/layouts/case-studies/list.html | 2 +-
.../partials/section-menu/en/documentation.html | 1 +
205 files changed, 5469 insertions(+), 1938 deletions(-)
create mode 100644 sdks/go/pkg/beam/runners/prism/internal/execute_test.go
create mode 100644 sdks/go/pkg/beam/runners/prism/internal/separate_test.go
create mode 100644 sdks/go/pkg/beam/runners/prism/internal/stage.go
create mode 100644 sdks/go/pkg/beam/runners/prism/prism.go
delete mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/util/ByteBuddyUtils.java
create mode 100644 sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/coders/DefaultCoderTest.java
create mode 100644 sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/SchemaCoderTest.java
create mode 100644 sdks/java/extensions/avro/src/test/java/org/apache/beam/sdk/extensions/avro/schemas/transforms/ConvertTest.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/debug/DataSampler.java
create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/debug/OutputSampler.java
copy sdks/java/{io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/changestreams/model/ModType.java => harness/src/main/java/org/apache/beam/fn/harness/debug/package-info.java} (66%)
create mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/debug/DataSamplerTest.java
create mode 100644 sdks/java/harness/src/test/java/org/apache/beam/fn/harness/debug/OutputSamplerTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryStorageStreamBundleSource.java
copy sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/{BigQueryIOStorageReadTest.java => BigQueryIOStorageReadWithStreamBundleSourceTest.java} (64%)
create mode 100644 sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcSchemaIOProviderTest.java
create mode 100644 sdks/python/apache_beam/examples/inference/tensorflow_mnist_with_weights.py
create mode 100755 website/www/site/content/en/documentation/ml/model-evaluation.md
mode change 100644 => 100755 website/www/site/layouts/partials/section-menu/en/documentation.html