You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ah...@apache.org on 2025/03/20 20:28:56 UTC
(beam) branch master updated (9e1cf5aaee4 -> 332fe03b728)
This is an automated email from the ASF dual-hosted git repository.
ahmedabualsaud pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 9e1cf5aaee4 Kafta Table consumer properties feature for beamSQL (#34366)
add 332fe03b728 [Managed Iceberg] unbounded source (#33504)
No new revisions were added by this update.
Summary of changes:
.../IO_Iceberg_Integration_Tests.json | 2 +-
CHANGES.md | 1 +
contributor-docs/discussion-docs/2025.md | 8 +-
.../model/pipeline/v1/external_transforms.proto | 2 +
sdks/java/io/iceberg/build.gradle | 3 +-
.../beam/sdk/io/iceberg/AppendFilesToTables.java | 2 +-
.../beam/sdk/io/iceberg/CreateReadTasksDoFn.java | 112 +++++++
.../beam/sdk/io/iceberg/IcebergCatalogConfig.java | 37 ++-
.../IcebergCdcReadSchemaTransformProvider.java | 215 +++++++++++++
.../org/apache/beam/sdk/io/iceberg/IcebergIO.java | 358 ++++++++++++++++++---
.../IcebergReadSchemaTransformProvider.java | 61 +++-
.../beam/sdk/io/iceberg/IcebergScanConfig.java | 106 +++++-
.../iceberg/IcebergSchemaTransformTranslation.java | 15 +
.../IcebergWriteSchemaTransformProvider.java | 5 +-
.../beam/sdk/io/iceberg/IncrementalScanSource.java | 102 ++++++
.../apache/beam/sdk/io/iceberg/ReadFromTasks.java | 98 ++++++
.../org/apache/beam/sdk/io/iceberg/ReadTask.java | 91 ++++++
.../beam/sdk/io/iceberg/ReadTaskDescriptor.java} | 38 ++-
.../org/apache/beam/sdk/io/iceberg/ReadUtils.java | 194 +++++++++++
.../apache/beam/sdk/io/iceberg/ScanTaskReader.java | 24 +-
.../io/iceberg/SchemaTransformConfiguration.java | 69 ----
.../apache/beam/sdk/io/iceberg/SnapshotInfo.java | 58 +++-
.../org/apache/beam/sdk/io/iceberg/TableCache.java | 86 +++++
.../beam/sdk/io/iceberg/WatchForSnapshots.java | 195 +++++++++++
...IcebergCdcReadSchemaTransformProviderTest.java} | 124 +++----
.../beam/sdk/io/iceberg/IcebergIOReadTest.java | 339 ++++++++++++++-----
.../IcebergReadSchemaTransformProviderTest.java | 75 ++---
.../IcebergSchemaTransformTranslationTest.java | 148 +++++++--
.../apache/beam/sdk/io/iceberg/ReadUtilsTest.java | 241 ++++++++++++++
.../beam/sdk/io/iceberg/TestDataWarehouse.java | 60 ++++
.../apache/beam/sdk/io/iceberg/TestFixtures.java | 58 ++--
.../io/iceberg/catalog/IcebergCatalogBaseIT.java | 153 ++++++++-
.../java/org/apache/beam/sdk/managed/Managed.java | 22 +-
sdks/standard_expansion_services.yaml | 1 +
34 files changed, 2640 insertions(+), 463 deletions(-)
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/CreateReadTasksDoFn.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergCdcReadSchemaTransformProvider.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IncrementalScanSource.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/ReadFromTasks.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/ReadTask.java
copy sdks/java/io/{mqtt/src/main/java/org/apache/beam/sdk/io/mqtt/MqttRecord.java => iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/ReadTaskDescriptor.java} (54%)
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/ReadUtils.java
delete mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SchemaTransformConfiguration.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/TableCache.java
create mode 100644 sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/WatchForSnapshots.java
copy sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/{IcebergReadSchemaTransformProviderTest.java => IcebergCdcReadSchemaTransformProviderTest.java} (62%)
create mode 100644 sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/ReadUtilsTest.java