You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2024/04/10 04:43:24 UTC
(beam) branch nightly-refs/heads/master updated (f39a179fbf3 -> 06588745182)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from f39a179fbf3 Bump github.com/aws/smithy-go from 1.20.1 to 1.20.2 in /sdks (#30883)
add 812e98fac24 Bump GCP-BOM to 26.36.0 (#30868)
add 0437a8dcfcc Initial Iceberg connector
add fd08eb49f31 Fix up IcebergIO Write path
add a7a6515cf3f Add IcebergIO GitHub Action workflow
add 819e54c972c Merge pull request #30797: Initial Iceberg Sink
add e3ce3bca115 Update Python Dependencies (#30816)
add 54c2a7277b4 [Flink] Speed up file write in batch mode by using larger bundle size (#30802)
add 76710e8e734 Implement LockFreeHistogram and use it for PerWorkerHistograms (#30769)
add fcc37a0cc2c [yaml] remove Combine from yaml_experimental_features (#30897)
add 282ff87726d Add another dependency restriction to unblock tests. (#30907)
add 3f4b2561c58 [bug30870]: make consumer polling timeout configurable for KafkaIO.Read (#30877)
add 06588745182 [yaml] disable javascript mapping for python >=3.12 (#30843)
No new revisions were added by this update.
Summary of changes:
.github/workflows/IO_Iceberg.yml | 118 +++++++
CHANGES.md | 1 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 20 +-
.../get-started/try-apache-beam-yaml.ipynb | 6 +-
.../beam/runners/flink/FlinkPipelineOptions.java | 42 ++-
.../runners/flink/FlinkTransformOverrides.java | 23 +-
.../runners/flink/FlinkPipelineOptionsTest.java | 12 +-
.../runners/dataflow/worker/LockFreeHistogram.java | 223 +++++++++++++
.../MetricsToPerStepNamespaceMetricsConverter.java | 38 +--
.../worker/StreamingStepMetricsContainer.java | 15 +-
.../dataflow/worker/LockFreeHistogramTest.java | 185 +++++++++++
...ricsToPerStepNamespaceMetricsConverterTest.java | 60 ++--
.../worker/StreamingStepMetricsContainerTest.java | 19 +-
scripts/tools/bomupgrader.py | 2 +-
.../container/license_scripts/dep_urls_java.yaml | 2 +-
.../io/gcp/bigtable/BigtableServiceImplTest.java | 2 +-
sdks/java/io/iceberg/build.gradle | 98 ++++++
.../beam/io/iceberg/AppendFilesToTables.java | 102 ++++++
.../apache/beam/io/iceberg/AssignDestinations.java | 65 ++++
.../beam/io/iceberg/DynamicDestinations.java | 36 +++
.../apache/beam/io/iceberg/FileWriteResult.java | 203 ++++++++++++
.../beam/io/iceberg/IcebergCatalogConfig.java | 233 ++++++++++++++
.../apache/beam/io/iceberg/IcebergDestination.java | 65 ++++
.../java/org/apache/beam/io/iceberg/IcebergIO.java | 50 +++
.../beam/io/iceberg/IcebergTableCreateConfig.java | 50 +++
.../apache/beam/io/iceberg/IcebergWriteResult.java | 65 ++++
.../io/iceberg/OneTableDynamicDestinations.java | 65 ++++
.../apache/beam/io/iceberg/PropertyBuilder.java | 38 +++
.../org/apache/beam/io/iceberg/RecordWriter.java | 99 ++++++
.../java/org/apache/beam/io/iceberg/RowHelper.java | 100 ++++++
.../org/apache/beam/io/iceberg/SchemaHelper.java | 116 +++++++
.../beam/io/iceberg/WriteGroupedRowsToFiles.java | 103 ++++++
.../beam/io/iceberg/WriteToDestinations.java | 123 ++++++++
.../beam/io/iceberg/WriteUngroupedRowsToFiles.java | 350 +++++++++++++++++++++
.../org/apache/beam/io/iceberg/package-info.java | 20 ++
.../beam/io/iceberg/FileWriteResultTest.java | 147 +++++++++
.../apache/beam/io/iceberg/IcebergIOWriteTest.java | 293 +++++++++++++++++
.../org/apache/beam/io/iceberg/RowHelperTest.java | 128 ++++++++
.../apache/beam/io/iceberg/TestDataWarehouse.java | 146 +++++++++
.../org/apache/beam/io/iceberg/TestFixtures.java | 101 ++++++
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 35 ++-
.../KafkaIOReadImplementationCompatibility.java | 1 +
.../beam/sdk/io/kafka/ReadFromKafkaDoFn.java | 16 +-
.../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 12 +
.../beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java | 14 +
sdks/python/apache_beam/yaml/examples/README.md | 9 -
.../aggregation/combine_count_minimal.yaml | 3 -
.../aggregation/combine_max_minimal.yaml | 3 -
.../aggregation/combine_mean_minimal.yaml | 3 -
.../aggregation/combine_min_minimal.yaml | 3 -
.../aggregation/combine_multiple_aggregations.yaml | 3 -
.../transforms/aggregation/combine_sum.yaml | 3 -
.../aggregation/combine_sum_minimal.yaml | 3 -
.../transforms/aggregation/group_into_batches.yaml | 3 -
.../aggregation/top_largest_per_key.yaml | 3 -
.../aggregation/top_smallest_per_key.yaml | 3 -
.../yaml/examples/wordcount_minimal.yaml | 3 -
.../apache_beam/yaml/programming_guide_test.py | 15 +-
sdks/python/apache_beam/yaml/readme_test.py | 5 +-
sdks/python/apache_beam/yaml/yaml_combine.py | 3 -
sdks/python/apache_beam/yaml/yaml_combine_test.py | 21 +-
sdks/python/apache_beam/yaml/yaml_mapping.py | 26 +-
.../python/apache_beam/yaml/yaml_transform_test.py | 3 +-
sdks/python/apache_beam/yaml/yaml_udf_test.py | 10 +
.../container/py310/base_image_requirements.txt | 25 +-
.../container/py311/base_image_requirements.txt | 25 +-
.../container/py38/base_image_requirements.txt | 25 +-
.../container/py39/base_image_requirements.txt | 25 +-
sdks/python/setup.py | 3 +-
sdks/python/tox.ini | 2 +
settings.gradle.kts | 2 +
.../content/en/documentation/sdks/yaml-combine.md | 2 -
72 files changed, 3658 insertions(+), 218 deletions(-)
create mode 100644 .github/workflows/IO_Iceberg.yml
create mode 100644 runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/LockFreeHistogram.java
create mode 100644 runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/LockFreeHistogramTest.java
create mode 100644 sdks/java/io/iceberg/build.gradle
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/AppendFilesToTables.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/AssignDestinations.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/DynamicDestinations.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/FileWriteResult.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/IcebergCatalogConfig.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/IcebergDestination.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/IcebergIO.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/IcebergTableCreateConfig.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/IcebergWriteResult.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/OneTableDynamicDestinations.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/PropertyBuilder.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/RecordWriter.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/RowHelper.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/SchemaHelper.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/WriteGroupedRowsToFiles.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/WriteToDestinations.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/WriteUngroupedRowsToFiles.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/io/iceberg/package-info.java
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/io/iceberg/FileWriteResultTest.java
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/io/iceberg/IcebergIOWriteTest.java
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/io/iceberg/RowHelperTest.java
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/io/iceberg/TestDataWarehouse.java
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/io/iceberg/TestFixtures.java