You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2023/02/04 05:13:41 UTC
[beam] branch nightly-refs/heads/master updated (49f00ab5c8d -> 679d30256c6)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 49f00ab5c8d [CdapIO] Implement windowed write (#25206)
add d50924d2ae9 [Spark Dataset runner] Fix collection encoder bug that may lead to corrupted data due to naive usage of unsafe storage (fixes #25296). (#25297)
add 72781cb089a [Spark Dataset runner] Break linage of dataset to reduce Spark planning overhead in case of large complex query plans (relates to #24710 and #23845) (#25187)
add 86480f33397 Fix flaky test due to create bigquery dataset conflict (#25266)
add b3be8b65aff 24472 Implement FileWriteSchemaTransformProvider (#24806)
add 0441d9d5823 Embed ML video to docs (#25302)
add d68b02804a5 skip automated expansion test (#25304)
add ed699f4504f Add banner highlighting beam ml (#25306)
add b1c9d8aec07 Optimize to use cached output receiver instead of creating one on DoFn invocation #21250 (#25245)
add c65e777d6c6 Optimize PGBK table to only update cache when there is a large enough size change. #21250 (#25219)
add 679d30256c6 Swap setting a context from being on the hot path when we emit elements to only be done during bundle creation and teardown #21250 (#25291)
No new revisions were added by this update.
Summary of changes:
.test-infra/tools/stale_bq_datasets_cleaner.sh | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +-
.../translation/EvaluationContext.java | 10 +-
.../translation/PipelineTranslator.java | 97 +++-
.../translation/TransformTranslator.java | 13 +-
.../batch/CombineGloballyTranslatorBatch.java | 4 +
.../batch/CombineGroupedValuesTranslatorBatch.java | 4 +
.../batch/CombinePerKeyTranslatorBatch.java | 4 +
.../translation/batch/FlattenTranslatorBatch.java | 4 +
.../batch/GroupByKeyTranslatorBatch.java | 5 +-
.../translation/batch/ImpulseTranslatorBatch.java | 4 +
.../translation/batch/ParDoTranslatorBatch.java | 6 +-
.../batch/ReadSourceTranslatorBatch.java | 7 +-
.../batch/ReshuffleTranslatorBatch.java | 8 +
.../batch/WindowAssignTranslatorBatch.java | 4 +
.../translation/helpers/EncoderHelpers.java | 19 +-
.../go/pkg/beam/core/runtime/xlangx/expand_test.go | 2 +
.../beam/sdk/transforms/DoFnOutputReceivers.java | 2 +-
.../jmh/PrecombineGroupingTableBenchmark.java | 87 ++--
.../java/org/apache/beam/fn/harness/Caches.java | 43 +-
.../apache/beam/fn/harness/FnApiDoFnRunner.java | 310 ++++++++++++-
.../beam/fn/harness/PrecombineGroupingTable.java | 10 +-
.../fn/harness/control/ExecutionStateSampler.java | 109 ++++-
.../fn/harness/control/ProcessBundleHandler.java | 57 +--
.../harness/data/PCollectionConsumerRegistry.java | 74 +--
.../harness/data/PTransformFunctionRegistry.java | 26 +-
.../fn/harness/PrecombineGroupingTableTest.java | 33 +-
.../harness/control/ExecutionStateSamplerTest.java | 144 ++++++
.../harness/control/ProcessBundleHandlerTest.java | 11 +-
.../data/PCollectionConsumerRegistryTest.java | 145 +++---
.../data/PTransformFunctionRegistryTest.java | 81 ++--
.../beam/sdk/io/common/SchemaAwareJavaBeans.java | 494 +++++++++++++++++++++
.../sdk/io/common/SchemaAwareJavaBeansTest.java | 175 ++++++++
.../OWNERS | 3 +
.../build.gradle | 20 +-
.../AvroWriteSchemaTransformFormatProvider.java | 88 ++++
.../CsvWriteSchemaTransformFormatProvider.java} | 31 +-
.../FileWriteSchemaTransformConfiguration.java | 66 +--
.../FileWriteSchemaTransformFormatProvider.java | 16 +-
.../FileWriteSchemaTransformFormatProviders.java | 148 ++++++
.../FileWriteSchemaTransformProvider.java | 191 ++++++++
.../JsonWriteSchemaTransformFormatProvider.java | 93 ++++
.../ParquetWriteSchemaTransformFormatProvider.java | 117 +++++
.../fileschematransform/XmlDateTimeAdapter.java} | 36 +-
.../sdk/io/fileschematransform/XmlRowAdapter.java | 93 ++++
.../sdk/io/fileschematransform/XmlRowValue.java | 268 +++++++++++
.../XmlWriteSchemaTransformFormatProvider.java | 115 +++++
.../sdk/io/fileschematransform/package-info.java | 0
...FileWriteSchemaTransformFormatProviderTest.java | 104 +++++
...FileWriteSchemaTransformFormatProviderTest.java | 434 ++++++++++++++++++
...WriteSchemaTransformFormatProviderTestData.java | 231 ++++++++++
...ileWriteSchemaTransformFormatProvidersTest.java | 17 +-
.../FileWriteSchemaTransformProviderTest.java | 127 ++++++
...FileWriteSchemaTransformFormatProviderTest.java | 104 +++++
...FileWriteSchemaTransformFormatProviderTest.java | 112 +++++
.../XmlDateTimeAdapterTest.java} | 32 +-
...FileWriteSchemaTransformFormatProviderTest.java | 120 +++++
.../io/fileschematransform/XmlRowAdapterTest.java | 190 ++++++++
.../io/fileschematransform/XmlRowValueTest.java | 391 ++++++++++++++++
.../FileWriteSchemaTransformFormatProviders.java | 124 ------
.../io/gcp/big_query_query_to_table_it_test.py | 6 +-
.../apache_beam/io/gcp/bigquery_file_loads_test.py | 8 +-
.../apache_beam/io/gcp/bigquery_read_it_test.py | 10 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 14 +-
.../apache_beam/io/gcp/bigquery_write_it_test.py | 8 +-
sdks/python/apache_beam/io/gcp/tests/utils.py | 6 +-
settings.gradle.kts | 2 +-
website/www/site/assets/scss/_banner.sass | 43 ++
website/www/site/assets/scss/_hero.scss | 3 +-
website/www/site/assets/scss/_navbar-desktop.scss | 3 +-
website/www/site/assets/scss/main.scss | 1 +
.../site/content/en/documentation/ml/overview.md | 4 +-
.../get-started/resources/videos-and-podcasts.md | 7 +
website/www/site/layouts/partials/header.html | 6 +
website/www/site/static/images/banner_desktop.jpg | Bin 0 -> 111671 bytes
website/www/site/static/images/banner_mobile.jpg | Bin 0 -> 59518 bytes
76 files changed, 4746 insertions(+), 642 deletions(-)
create mode 100644 sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/SchemaAwareJavaBeans.java
create mode 100644 sdks/java/io/common/src/test/java/org/apache/beam/sdk/io/common/SchemaAwareJavaBeansTest.java
rename sdks/java/io/{fileschematransform => file-schema-transform}/OWNERS (55%)
rename sdks/java/io/{fileschematransform => file-schema-transform}/build.gradle (70%)
create mode 100644 sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/AvroWriteSchemaTransformFormatProvider.java
copy sdks/java/io/{fileschematransform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvider.java => file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/CsvWriteSchemaTransformFormatProvider.java} (56%)
rename sdks/java/io/{fileschematransform => file-schema-transform}/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformConfiguration.java (77%)
rename sdks/java/io/{fileschematransform => file-schema-transform}/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvider.java (66%)
create mode 100644 sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviders.java
create mode 100644 sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformProvider.java
create mode 100644 sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/JsonWriteSchemaTransformFormatProvider.java
create mode 100644 sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/ParquetWriteSchemaTransformFormatProvider.java
copy sdks/java/io/{fileschematransform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvidersTest.java => file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/XmlDateTimeAdapter.java} (51%)
create mode 100644 sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/XmlRowAdapter.java
create mode 100644 sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/XmlRowValue.java
create mode 100644 sdks/java/io/file-schema-transform/src/main/java/org/apache/beam/sdk/io/fileschematransform/XmlWriteSchemaTransformFormatProvider.java
rename sdks/java/io/{fileschematransform => file-schema-transform}/src/main/java/org/apache/beam/sdk/io/fileschematransform/package-info.java (100%)
create mode 100644 sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/AvroFileWriteSchemaTransformFormatProviderTest.java
create mode 100644 sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviderTest.java
create mode 100644 sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviderTestData.java
copy sdks/java/io/{fileschematransform => file-schema-transform}/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvidersTest.java (64%)
create mode 100644 sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformProviderTest.java
create mode 100644 sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/JsonFileWriteSchemaTransformFormatProviderTest.java
create mode 100644 sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/ParquetFileWriteSchemaTransformFormatProviderTest.java
rename sdks/java/io/{fileschematransform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProvidersTest.java => file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/XmlDateTimeAdapterTest.java} (56%)
create mode 100644 sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/XmlFileWriteSchemaTransformFormatProviderTest.java
create mode 100644 sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/XmlRowAdapterTest.java
create mode 100644 sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/XmlRowValueTest.java
delete mode 100644 sdks/java/io/fileschematransform/src/main/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviders.java
create mode 100644 website/www/site/assets/scss/_banner.sass
create mode 100644 website/www/site/static/images/banner_desktop.jpg
create mode 100644 website/www/site/static/images/banner_mobile.jpg