You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@xtable.apache.org by as...@apache.org on 2025/03/05 23:44:41 UTC
(incubator-xtable) branch 345-read-and-translate-the-deletion-vectors-in-delta-source-table-to-xtables-internal-representation updated (2e234ae9 -> 5036f3b5)
This is an automated email from the ASF dual-hosted git repository.
ashvin pushed a change to branch 345-read-and-translate-the-deletion-vectors-in-delta-source-table-to-xtables-internal-representation
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
omit 2e234ae9 Refactor: rename to clariy delete ordinal only
omit b65080ee Add support for inline delete vectors
omit 92712a6b Fix spotless error, unused import
omit 7786c5bd Handle optional offset information
omit 5f4f8515 Extract Delta Lake deletion vectors
add 14967dd6 Add community sync page for XTable
add b463ae6e Update GMeet link and add today's recording
add c9deec1a [590] Add RFC for XCatalogSync - Synchronize tables across catalogs
add 9fa812e3 [590] Add RunCatalogSync utility for synchronizing tables across catalogs
add d3bbe5fe Delta source: add record count for DataFile
add 79c68def [site] Update blogs
add 8b0b5eaf [590] Add Iceberg Glue Catalog Sync implementation
add 30f3e544 [590] Add Iceberg HMS Catalog Sync implementation
add f2045be9 [590] Add Delta Glue Catalog Sync implementation
add e79a2d34 [590] Add Delta HMS Catalog Sync implementation
add 2e71e156 [590] Changes in xtable-core for Hudi Catalog Sync
add 67946b98 Update conversion sources
add b3e03e55 [40] Add commit-level metadata and source-target id mapping
add f194f4cb [590] Add Hudi HMS Catalog Sync Implementation
add 03e5b022 [590] Add Hudi Glue Catalog Sync Implementation
add dc209962 Add change proposal for conversion of deletion vectors
add c76692b8 Add base representation of storage files
new 5036f3b5 Extract Delta Lake deletion vectors
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (2e234ae9)
\
N -- N -- N refs/heads/345-read-and-translate-the-deletion-vectors-in-delta-source-table-to-xtables-internal-representation (5036f3b5)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 2 +-
assets/images/catalog_sync_flow.jpg | Bin 0 -> 133357 bytes
pom.xml | 20 +-
rfc/rfc-1/rfc-1.md | 139 ++++
rfc/rfc-2/2 - Deletion Info Conversion.md | 227 ++++++
website/README.md | 4 +
website/blog/OneTable-is-now-Apache-XTable.md | 1 +
website/blog/xtable-in-aws-lambda.mdx | 19 +
website/community/sync.md | 34 +
website/docusaurus.config.js | 9 +
.../xtable-in-aws-lambda/xtable-in-aws-lambda.png | Bin 0 -> 115121 bytes
website/static/index.html | 2 +
.../apache/xtable/conversion/ConversionConfig.java | 12 +-
.../xtable/conversion/ExternalCatalogConfig.java | 65 ++
.../xtable/conversion/TargetCatalogConfig.java | 21 +-
.../org/apache/xtable/model/InternalSnapshot.java | 3 +
.../java/org/apache/xtable/model/TableChange.java | 23 +-
.../apache/xtable/model/exception/ErrorCode.java | 3 +-
.../xtable/model/metadata/TableSyncMetadata.java | 22 +-
.../org/apache/xtable/model/stat/FileStats.java | 15 +-
.../storage/{FileFormat.java => CatalogType.java} | 10 +-
.../org/apache/xtable/model/storage/FilesDiff.java | 6 +-
.../xtable/model/storage/InternalDataFile.java | 19 +-
.../model/storage/InternalDeletionVector.java | 26 +-
.../apache/xtable/model/storage/InternalFile.java | 56 ++
.../{DataFilesDiff.java => InternalFilesDiff.java} | 38 +-
.../xtable/model/storage/PartitionFileGroup.java | 12 +-
.../spi/extractor/CatalogConversionSource.java | 9 +
.../xtable/spi/extractor/ConversionSource.java | 14 +
.../org/apache/xtable/spi/sync/CatalogSync.java | 12 +-
.../apache/xtable/spi/sync/CatalogSyncClient.java | 9 +
.../apache/xtable/spi/sync/ConversionTarget.java | 16 +-
.../apache/xtable/spi/sync/TableFormatSync.java | 23 +-
.../model/metadata/TestTableSyncMetadata.java | 20 +-
.../xtable/model/storage/TestDataFilesDiff.java | 8 +-
.../apache/xtable/model/storage/TestFilesDiff.java | 2 +-
.../spi/extractor/TestExtractFromSource.java | 20 +-
.../xtable/spi/sync/TestTableFormatSync.java | 170 +++--
{xtable-api => xtable-aws}/pom.xml | 65 +-
.../xtable/glue/DefaultGlueClientFactory.java | 80 +++
.../org/apache/xtable/glue/GlueCatalogConfig.java | 102 +++
.../xtable/glue/GlueCatalogConversionSource.java | 100 +++
.../glue/GlueCatalogPartitionSyncOperations.java | 331 +++++++++
.../apache/xtable/glue/GlueCatalogSyncClient.java | 268 +++++++
.../glue/GlueCatalogTableBuilderFactory.java | 48 ++
.../apache/xtable/glue/GlueCatalogTableUtils.java | 56 ++
.../org/apache/xtable/glue/GlueClientFactory.java | 23 +-
.../apache/xtable/glue/GlueSchemaExtractor.java | 272 +++++++
.../glue/table/DeltaGlueCatalogTableBuilder.java | 116 +++
.../glue/table/HudiGlueCatalogTableBuilder.java | 185 +++++
.../glue/table/IcebergGlueCatalogTableBuilder.java | 121 ++++
...he.xtable.spi.extractor.CatalogConversionSource | 5 +-
.../org.apache.xtable.spi.sync.CatalogSyncClient | 5 +-
.../xtable/glue/GlueCatalogSyncTestBase.java | 238 ++++++
.../apache/xtable/glue/TestGlueCatalogConfig.java | 125 ++++
.../glue/TestGlueCatalogConversionSource.java | 193 +++++
.../TestGlueCatalogPartitionSyncOperations.java | 519 +++++++++++++
.../xtable/glue/TestGlueCatalogSyncClient.java | 463 ++++++++++++
.../xtable/glue/TestGlueSchemaExtractor.java | 799 +++++++++++++++++++++
.../table/TestDeltaGlueCatalogTableBuilder.java | 106 +++
.../table/TestHudiGlueCatalogTableBuilder.java | 125 ++++
.../table/TestIcebergGlueCatalogTableBuilder.java | 150 ++++
xtable-core/pom.xml | 30 +
.../xtable/catalog/CatalogConversionFactory.java | 102 +++
.../apache/xtable/catalog/CatalogPartition.java | 31 +-
.../xtable/catalog/CatalogPartitionEvent.java | 49 ++
.../catalog/CatalogPartitionSyncOperations.java | 99 +++
.../xtable/catalog/CatalogPartitionSyncTool.java | 25 +-
.../apache/xtable/catalog/CatalogTableBuilder.java | 20 +-
.../org/apache/xtable/catalog/CatalogUtils.java | 69 ++
.../java/org/apache/xtable/catalog/Constants.java | 49 ++
.../apache/xtable/catalog/TableFormatUtils.java | 89 +++
.../xtable/conversion/ConversionController.java | 212 ++++--
.../apache/xtable/conversion/ConversionUtils.java | 23 +-
.../apache/xtable/delta/DeltaActionsConverter.java | 19 +-
.../apache/xtable/delta/DeltaConversionSource.java | 24 +-
.../apache/xtable/delta/DeltaConversionTarget.java | 74 +-
.../delta/DeltaDataFileUpdatesExtractor.java | 19 +-
.../apache/xtable/delta/DeltaSchemaExtractor.java | 96 ---
.../apache/xtable/delta/DeltaStatsExtractor.java | 52 +-
...eadException.java => CatalogSyncException.java} | 20 +-
.../xtable/hudi/BaseFileUpdatesExtractor.java | 17 +-
.../apache/xtable/hudi/HudiConversionSource.java | 7 +
.../apache/xtable/hudi/HudiConversionTarget.java | 76 +-
.../apache/xtable/hudi/HudiDataFileExtractor.java | 6 +-
.../org/apache/xtable/hudi/HudiTableManager.java | 4 +-
.../hudi/catalog/HudiCatalogPartitionSyncTool.java | 471 ++++++++++++
.../HudiCatalogTablePropertiesExtractor.java | 148 ++++
.../xtable/hudi/catalog/HudiInputFormatUtils.java | 112 +++
.../xtable/iceberg/IcebergCatalogConfig.java | 8 +-
.../xtable/iceberg/IcebergConversionSource.java | 21 +-
.../xtable/iceberg/IcebergConversionTarget.java | 45 +-
.../xtable/iceberg/IcebergDataFileUpdatesSync.java | 40 +-
.../apache/xtable/reflection/ReflectionUtils.java | 42 +-
.../apache/xtable/schema/SparkSchemaExtractor.java | 133 ++++
.../org/apache/xtable/ValidationTestHelper.java | 14 +-
.../catalog/TestCatalogConversionFactory.java | 108 +++
.../apache/xtable/catalog/TestCatalogUtils.java | 65 ++
.../xtable/catalog/TestSchemaExtractorBase.java | 92 +++
.../xtable/catalog/TestTableFormatUtils.java | 102 +++
.../conversion/TestConversionController.java | 231 +++++-
.../xtable/delta/ITDeltaConversionSource.java | 2 +-
.../xtable/delta/ITDeltaDeleteVectorConvert.java | 12 +-
.../xtable/delta/TestDeltaActionsConverter.java | 18 +-
.../xtable/delta/TestDeltaSchemaExtractor.java | 134 ----
.../xtable/delta/TestDeltaStatsExtractor.java | 29 +-
.../org/apache/xtable/delta/TestDeltaSync.java | 78 +-
.../apache/xtable/hudi/ITHudiConversionTarget.java | 174 ++++-
.../xtable/hudi/TestBaseFileUpdatesExtractor.java | 6 +-
.../xtable/hudi/TestHudiConversionTarget.java | 7 +-
.../catalog/TestHudiCatalogPartitionSyncTool.java | 323 +++++++++
.../TestHudiCatalogTablePropertiesExtractor.java | 182 +++++
.../hudi/catalog/TestHudiInputFormatUtils.java | 118 +++
.../iceberg/TestIcebergConversionSource.java | 4 +-
.../org/apache/xtable/iceberg/TestIcebergSync.java | 123 +++-
.../TestSparkSchemaExtractor.java} | 258 +------
.../org/apache/xtable/testutil/ITTestUtils.java | 117 +++
...e.xtable.spi.extractor.CatalogConversionSource} | 5 +-
.../org.apache.xtable.spi.sync.CatalogSyncClient} | 5 +-
xtable-hive-metastore/pom.xml | 207 ++++++
.../org/apache/xtable/hms/HMSCatalogConfig.java | 65 ++
.../xtable/hms/HMSCatalogConversionSource.java | 111 +++
.../hms/HMSCatalogPartitionSyncOperations.java | 221 ++++++
.../apache/xtable/hms/HMSCatalogSyncClient.java | 240 +++++++
.../xtable/hms/HMSCatalogTableBuilderFactory.java | 77 ++
.../org/apache/xtable/hms/HMSClientProvider.java | 65 ++
.../org/apache/xtable/hms/HMSSchemaExtractor.java | 143 ++++
.../hms/table/DeltaHMSCatalogTableBuilder.java | 95 +++
.../hms/table/HudiHMSCatalogTableBuilder.java | 185 +++++
.../hms/table/IcebergHMSCatalogTableBuilder.java | 126 ++++
...he.xtable.spi.extractor.CatalogConversionSource | 5 +-
.../org.apache.xtable.spi.sync.CatalogSyncClient | 5 +-
.../apache/xtable/hms/HMSCatalogSyncTestBase.java | 188 +++++
.../apache/xtable/hms/TestHMSCatalogConfig.java | 64 ++
.../xtable/hms/TestHMSCatalogConversionSource.java | 181 +++++
.../hms/TestHMSCatalogPartitionSyncOperations.java | 470 ++++++++++++
.../xtable/hms/TestHMSCatalogSyncClient.java | 420 +++++++++++
.../hms/TestHMSCatalogTableBuilderFactory.java | 64 ++
.../apache/xtable/hms/TestHMSSchemaExtractor.java | 493 +++++++++++++
.../hms/table/TestDeltaHMSCatalogTableBuilder.java | 134 ++++
.../hms/table/TestHudiHMSCatalogTableBuilder.java | 107 +++
.../table/TestIcebergHMSCatalogTableBuilder.java | 185 +++++
xtable-utilities/pom.xml | 49 ++
.../apache/xtable/utilities/RunCatalogSync.java | 381 ++++++++++
.../java/org/apache/xtable/utilities/RunSync.java | 4 +-
.../apache/xtable/utilities/ITRunCatalogSync.java | 142 ++++
.../xtable/utilities/TestRunCatalogSync.java | 21 +-
.../src/test/resources/catalogConfig.yaml | 72 ++
148 files changed, 13568 insertions(+), 977 deletions(-)
create mode 100644 assets/images/catalog_sync_flow.jpg
create mode 100644 rfc/rfc-1/rfc-1.md
create mode 100644 rfc/rfc-2/2 - Deletion Info Conversion.md
create mode 100644 website/blog/xtable-in-aws-lambda.mdx
create mode 100644 website/community/sync.md
create mode 100644 website/static/images/blog/xtable-in-aws-lambda/xtable-in-aws-lambda.png
create mode 100644 xtable-api/src/main/java/org/apache/xtable/conversion/ExternalCatalogConfig.java
copy xtable-core/src/main/java/org/apache/xtable/iceberg/IcebergCatalogConfig.java => xtable-api/src/main/java/org/apache/xtable/conversion/TargetCatalogConfig.java (64%)
copy xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStats.java => xtable-api/src/main/java/org/apache/xtable/model/stat/FileStats.java (82%)
copy xtable-api/src/main/java/org/apache/xtable/model/storage/{FileFormat.java => CatalogType.java} (80%)
create mode 100644 xtable-api/src/main/java/org/apache/xtable/model/storage/InternalFile.java
rename xtable-api/src/main/java/org/apache/xtable/model/storage/{DataFilesDiff.java => InternalFilesDiff.java} (62%)
copy {xtable-api => xtable-aws}/pom.xml (64%)
create mode 100644 xtable-aws/src/main/java/org/apache/xtable/glue/DefaultGlueClientFactory.java
create mode 100644 xtable-aws/src/main/java/org/apache/xtable/glue/GlueCatalogConfig.java
create mode 100644 xtable-aws/src/main/java/org/apache/xtable/glue/GlueCatalogConversionSource.java
create mode 100644 xtable-aws/src/main/java/org/apache/xtable/glue/GlueCatalogPartitionSyncOperations.java
create mode 100644 xtable-aws/src/main/java/org/apache/xtable/glue/GlueCatalogSyncClient.java
create mode 100644 xtable-aws/src/main/java/org/apache/xtable/glue/GlueCatalogTableBuilderFactory.java
create mode 100644 xtable-aws/src/main/java/org/apache/xtable/glue/GlueCatalogTableUtils.java
copy xtable-api/src/main/java/org/apache/xtable/model/storage/DataLayoutStrategy.java => xtable-aws/src/main/java/org/apache/xtable/glue/GlueClientFactory.java (65%)
create mode 100644 xtable-aws/src/main/java/org/apache/xtable/glue/GlueSchemaExtractor.java
create mode 100644 xtable-aws/src/main/java/org/apache/xtable/glue/table/DeltaGlueCatalogTableBuilder.java
create mode 100644 xtable-aws/src/main/java/org/apache/xtable/glue/table/HudiGlueCatalogTableBuilder.java
create mode 100644 xtable-aws/src/main/java/org/apache/xtable/glue/table/IcebergGlueCatalogTableBuilder.java
copy xtable-core/src/main/resources/META-INF/services/org.apache.xtable.spi.sync.ConversionTarget => xtable-aws/src/main/resources/META-INF/services/org.apache.xtable.spi.extractor.CatalogConversionSource (86%)
copy xtable-core/src/main/resources/META-INF/services/org.apache.xtable.spi.sync.ConversionTarget => xtable-aws/src/main/resources/META-INF/services/org.apache.xtable.spi.sync.CatalogSyncClient (86%)
create mode 100644 xtable-aws/src/test/java/org/apache/xtable/glue/GlueCatalogSyncTestBase.java
create mode 100644 xtable-aws/src/test/java/org/apache/xtable/glue/TestGlueCatalogConfig.java
create mode 100644 xtable-aws/src/test/java/org/apache/xtable/glue/TestGlueCatalogConversionSource.java
create mode 100644 xtable-aws/src/test/java/org/apache/xtable/glue/TestGlueCatalogPartitionSyncOperations.java
create mode 100644 xtable-aws/src/test/java/org/apache/xtable/glue/TestGlueCatalogSyncClient.java
create mode 100644 xtable-aws/src/test/java/org/apache/xtable/glue/TestGlueSchemaExtractor.java
create mode 100644 xtable-aws/src/test/java/org/apache/xtable/glue/table/TestDeltaGlueCatalogTableBuilder.java
create mode 100644 xtable-aws/src/test/java/org/apache/xtable/glue/table/TestHudiGlueCatalogTableBuilder.java
create mode 100644 xtable-aws/src/test/java/org/apache/xtable/glue/table/TestIcebergGlueCatalogTableBuilder.java
create mode 100644 xtable-core/src/main/java/org/apache/xtable/catalog/CatalogConversionFactory.java
copy xtable-api/src/main/java/org/apache/xtable/model/storage/DataLayoutStrategy.java => xtable-core/src/main/java/org/apache/xtable/catalog/CatalogPartition.java (52%)
create mode 100644 xtable-core/src/main/java/org/apache/xtable/catalog/CatalogPartitionEvent.java
create mode 100644 xtable-core/src/main/java/org/apache/xtable/catalog/CatalogPartitionSyncOperations.java
copy xtable-api/src/main/java/org/apache/xtable/spi/extractor/CatalogConversionSource.java => xtable-core/src/main/java/org/apache/xtable/catalog/CatalogPartitionSyncTool.java (50%)
copy xtable-api/src/main/java/org/apache/xtable/model/validation/ValidationChecker.java => xtable-core/src/main/java/org/apache/xtable/catalog/CatalogTableBuilder.java (62%)
create mode 100644 xtable-core/src/main/java/org/apache/xtable/catalog/CatalogUtils.java
create mode 100644 xtable-core/src/main/java/org/apache/xtable/catalog/Constants.java
create mode 100644 xtable-core/src/main/java/org/apache/xtable/catalog/TableFormatUtils.java
copy xtable-api/src/main/java/org/apache/xtable/model/storage/TableFormat.java => xtable-core/src/main/java/org/apache/xtable/conversion/ConversionUtils.java (67%)
copy xtable-core/src/main/java/org/apache/xtable/exception/{ReadException.java => CatalogSyncException.java} (63%)
create mode 100644 xtable-core/src/main/java/org/apache/xtable/hudi/catalog/HudiCatalogPartitionSyncTool.java
create mode 100644 xtable-core/src/main/java/org/apache/xtable/hudi/catalog/HudiCatalogTablePropertiesExtractor.java
create mode 100644 xtable-core/src/main/java/org/apache/xtable/hudi/catalog/HudiInputFormatUtils.java
create mode 100644 xtable-core/src/main/java/org/apache/xtable/schema/SparkSchemaExtractor.java
create mode 100644 xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java
create mode 100644 xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogUtils.java
create mode 100644 xtable-core/src/test/java/org/apache/xtable/catalog/TestSchemaExtractorBase.java
create mode 100644 xtable-core/src/test/java/org/apache/xtable/catalog/TestTableFormatUtils.java
create mode 100644 xtable-core/src/test/java/org/apache/xtable/hudi/catalog/TestHudiCatalogPartitionSyncTool.java
create mode 100644 xtable-core/src/test/java/org/apache/xtable/hudi/catalog/TestHudiCatalogTablePropertiesExtractor.java
create mode 100644 xtable-core/src/test/java/org/apache/xtable/hudi/catalog/TestHudiInputFormatUtils.java
copy xtable-core/src/test/java/org/apache/xtable/{delta/TestDeltaSchemaExtractor.java => schema/TestSparkSchemaExtractor.java} (73%)
copy xtable-core/src/{main/resources/META-INF/services/org.apache.xtable.spi.sync.ConversionTarget => test/resources/META-INF/services/org.apache.xtable.spi.extractor.CatalogConversionSource} (86%)
copy xtable-core/src/{main/resources/META-INF/services/org.apache.xtable.spi.sync.ConversionTarget => test/resources/META-INF/services/org.apache.xtable.spi.sync.CatalogSyncClient} (86%)
create mode 100644 xtable-hive-metastore/pom.xml
create mode 100644 xtable-hive-metastore/src/main/java/org/apache/xtable/hms/HMSCatalogConfig.java
create mode 100644 xtable-hive-metastore/src/main/java/org/apache/xtable/hms/HMSCatalogConversionSource.java
create mode 100644 xtable-hive-metastore/src/main/java/org/apache/xtable/hms/HMSCatalogPartitionSyncOperations.java
create mode 100644 xtable-hive-metastore/src/main/java/org/apache/xtable/hms/HMSCatalogSyncClient.java
create mode 100644 xtable-hive-metastore/src/main/java/org/apache/xtable/hms/HMSCatalogTableBuilderFactory.java
create mode 100644 xtable-hive-metastore/src/main/java/org/apache/xtable/hms/HMSClientProvider.java
create mode 100644 xtable-hive-metastore/src/main/java/org/apache/xtable/hms/HMSSchemaExtractor.java
create mode 100644 xtable-hive-metastore/src/main/java/org/apache/xtable/hms/table/DeltaHMSCatalogTableBuilder.java
create mode 100644 xtable-hive-metastore/src/main/java/org/apache/xtable/hms/table/HudiHMSCatalogTableBuilder.java
create mode 100644 xtable-hive-metastore/src/main/java/org/apache/xtable/hms/table/IcebergHMSCatalogTableBuilder.java
copy xtable-core/src/main/resources/META-INF/services/org.apache.xtable.spi.sync.ConversionTarget => xtable-hive-metastore/src/main/resources/META-INF/services/org.apache.xtable.spi.extractor.CatalogConversionSource (86%)
copy xtable-core/src/main/resources/META-INF/services/org.apache.xtable.spi.sync.ConversionTarget => xtable-hive-metastore/src/main/resources/META-INF/services/org.apache.xtable.spi.sync.CatalogSyncClient (86%)
create mode 100644 xtable-hive-metastore/src/test/java/org/apache/xtable/hms/HMSCatalogSyncTestBase.java
create mode 100644 xtable-hive-metastore/src/test/java/org/apache/xtable/hms/TestHMSCatalogConfig.java
create mode 100644 xtable-hive-metastore/src/test/java/org/apache/xtable/hms/TestHMSCatalogConversionSource.java
create mode 100644 xtable-hive-metastore/src/test/java/org/apache/xtable/hms/TestHMSCatalogPartitionSyncOperations.java
create mode 100644 xtable-hive-metastore/src/test/java/org/apache/xtable/hms/TestHMSCatalogSyncClient.java
create mode 100644 xtable-hive-metastore/src/test/java/org/apache/xtable/hms/TestHMSCatalogTableBuilderFactory.java
create mode 100644 xtable-hive-metastore/src/test/java/org/apache/xtable/hms/TestHMSSchemaExtractor.java
create mode 100644 xtable-hive-metastore/src/test/java/org/apache/xtable/hms/table/TestDeltaHMSCatalogTableBuilder.java
create mode 100644 xtable-hive-metastore/src/test/java/org/apache/xtable/hms/table/TestHudiHMSCatalogTableBuilder.java
create mode 100644 xtable-hive-metastore/src/test/java/org/apache/xtable/hms/table/TestIcebergHMSCatalogTableBuilder.java
create mode 100644 xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java
create mode 100644 xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunCatalogSync.java
copy xtable-api/src/test/java/org/apache/xtable/conversion/TestTargetTable.java => xtable-utilities/src/test/java/org/apache/xtable/utilities/TestRunCatalogSync.java (64%)
create mode 100644 xtable-utilities/src/test/resources/catalogConfig.yaml
(incubator-xtable) 01/01: Extract Delta Lake deletion vectors
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ashvin pushed a commit to branch 345-read-and-translate-the-deletion-vectors-in-delta-source-table-to-xtables-internal-representation
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
commit 5036f3b55f3bfd7a4007d8f4efcca3f0666fd3fe
Author: Ashvin Agrawal <as...@apache.org>
AuthorDate: Tue Jan 21 21:00:09 2025 -0800
Extract Delta Lake deletion vectors
This change extracts deletion vectors represented as roaring bitmaps in delta lake files and
converts them into the XTable intermediate representation.
Previously, XTable only detected tables changes that included adding or removing of data files. Now
the detected table change also includes any deletion vectors files added in the commit.
Note that, in Delta Lake, the Deletion vectors are represented in a compressed binary format.
However, once extracted by Xtable, the offset are currently extracted into a list of long offsets.
This representation is not the most efficient for large datasets. Optimization is pending to
prioritize end-to-end conversion completion.
---
.../java/org/apache/xtable/model/TableChange.java | 2 +-
.../model/storage/InternalDeletionVector.java | 75 ++++++++
.../apache/xtable/delta/DeltaActionsConverter.java | 67 +++++++-
.../apache/xtable/delta/DeltaConversionSource.java | 25 +--
.../org/apache/xtable/TestSparkDeltaTable.java | 7 +-
.../org/apache/xtable/ValidationTestHelper.java | 6 +-
.../xtable/delta/ITDeltaDeleteVectorConvert.java | 189 ++++++++++++++++++---
.../xtable/delta/TestDeltaActionsConverter.java | 164 +++++++++++++++---
8 files changed, 478 insertions(+), 57 deletions(-)
diff --git a/xtable-api/src/main/java/org/apache/xtable/model/TableChange.java b/xtable-api/src/main/java/org/apache/xtable/model/TableChange.java
index b425fd01..287b8f38 100644
--- a/xtable-api/src/main/java/org/apache/xtable/model/TableChange.java
+++ b/xtable-api/src/main/java/org/apache/xtable/model/TableChange.java
@@ -30,7 +30,7 @@ import org.apache.xtable.model.storage.InternalFilesDiff;
* @since 0.1
*/
@Value
-@Builder(toBuilder = true)
+@Builder(toBuilder = true, builderClassName = "Builder")
public class TableChange {
// Change in files at the specified instant
InternalFilesDiff filesDiff;
diff --git a/xtable-api/src/main/java/org/apache/xtable/model/storage/InternalDeletionVector.java b/xtable-api/src/main/java/org/apache/xtable/model/storage/InternalDeletionVector.java
new file mode 100644
index 00000000..868749dc
--- /dev/null
+++ b/xtable-api/src/main/java/org/apache/xtable/model/storage/InternalDeletionVector.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.model.storage;
+
+import java.util.Iterator;
+import java.util.function.Supplier;
+
+import lombok.AccessLevel;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.ToString;
+import lombok.experimental.Accessors;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;
+
+@Accessors(fluent = true)
+@SuperBuilder(toBuilder = true)
+@FieldDefaults(makeFinal = true, level = lombok.AccessLevel.PRIVATE)
+@Getter
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+public class InternalDeletionVector extends InternalDataFile {
+ // path (absolute with scheme) of data file to which this deletion vector belongs
+ @NonNull String dataFilePath;
+
+ // super.getFileSizeBytes() is the size of the deletion vector file
+ // super.getPhysicalPath() is the absolute path (with scheme) of the deletion vector file
+ // super.getRecordCount() is the count of records in the deletion vector file
+
+ // offset of deletion vector start in a deletion vector file
+ int offset;
+
+ /**
+ * binary representation of the deletion vector. The consumer can use the {@link
+ * #ordinalsIterator()} to extract the ordinals represented in the binary format.
+ */
+ byte[] binaryRepresentation;
+
+ /**
+ * Supplier for an iterator that returns the ordinals of records deleted by this deletion vector
+ * in the linked data file, identified by {@link #dataFilePath}.
+ *
+ * <p>The {@link InternalDeletionVector} instance does not guarantee that a new or distinct result
+ * will be returned each time the supplier is invoked. However, the supplier is expected to return
+ * a new iterator for each call.
+ */
+ @Getter(AccessLevel.NONE)
+ Supplier<Iterator<Long>> ordinalsSupplier;
+
+ /**
+ * @return An iterator that returns the ordinals of records deleted by this deletion vector in the
+ * linked data file. There is no guarantee that a new or distinct iterator will be returned
+ * each time the iterator is invoked.
+ */
+ public Iterator<Long> ordinalsIterator() {
+ return ordinalsSupplier.get();
+ }
+}
diff --git a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaActionsConverter.java b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaActionsConverter.java
index 16a320f1..9e3b33cf 100644
--- a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaActionsConverter.java
+++ b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaActionsConverter.java
@@ -18,18 +18,26 @@
package org.apache.xtable.delta;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.spark.sql.delta.Snapshot;
import org.apache.spark.sql.delta.actions.AddFile;
import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor;
import org.apache.spark.sql.delta.actions.RemoveFile;
+import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray;
+import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore;
+import org.apache.spark.sql.delta.storage.dv.HadoopFileSystemDVStore;
+
+import com.google.common.annotations.VisibleForTesting;
import org.apache.xtable.exception.NotSupportedException;
import org.apache.xtable.model.schema.InternalField;
@@ -38,6 +46,7 @@ import org.apache.xtable.model.stat.ColumnStat;
import org.apache.xtable.model.stat.FileStats;
import org.apache.xtable.model.storage.FileFormat;
import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalDeletionVector;
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class DeltaActionsConverter {
@@ -113,16 +122,66 @@ public class DeltaActionsConverter {
*
* @param snapshot the commit snapshot
* @param addFile the add file action
- * @return the deletion vector representation (path of data file), or null if no deletion vector
- * is present
+ * @return the deletion vector representation, or null if no deletion vector is present
*/
- public String extractDeletionVectorFile(Snapshot snapshot, AddFile addFile) {
+ public InternalDeletionVector extractDeletionVector(Snapshot snapshot, AddFile addFile) {
DeletionVectorDescriptor deletionVector = addFile.deletionVector();
if (deletionVector == null) {
return null;
}
String dataFilePath = addFile.path();
- return getFullPathToFile(snapshot, dataFilePath);
+ dataFilePath = getFullPathToFile(snapshot, dataFilePath);
+
+ InternalDeletionVector.InternalDeletionVectorBuilder<?, ?> deleteVectorBuilder =
+ InternalDeletionVector.builder()
+ .recordCount(deletionVector.cardinality())
+ .fileSizeBytes(deletionVector.sizeInBytes())
+ .dataFilePath(dataFilePath);
+
+ if (deletionVector.isInline()) {
+ deleteVectorBuilder
+ .binaryRepresentation(deletionVector.inlineData())
+ .physicalPath("")
+ .ordinalsSupplier(() -> ordinalsIterator(deletionVector.inlineData()));
+ } else {
+ Path deletionVectorFilePath = deletionVector.absolutePath(snapshot.deltaLog().dataPath());
+ deleteVectorBuilder
+ .offset(getOffset(deletionVector))
+ .physicalPath(deletionVectorFilePath.toString())
+ .ordinalsSupplier(() -> ordinalsIterator(snapshot, deletionVector));
+ }
+
+ return deleteVectorBuilder.build();
+ }
+
+ private Iterator<Long> ordinalsIterator(byte[] bytes) {
+ RoaringBitmapArray rbm = RoaringBitmapArray.readFrom(bytes);
+ long[] ordinals = rbm.values();
+ return Arrays.stream(ordinals).iterator();
+ }
+
+ private Iterator<Long> ordinalsIterator(
+ Snapshot snapshot, DeletionVectorDescriptor deleteVector) {
+ Path deletionVectorFilePath = deleteVector.absolutePath(snapshot.deltaLog().dataPath());
+ int offset = getOffset(deleteVector);
+ long[] ordinals =
+ parseOrdinalFile(
+ snapshot.deltaLog().newDeltaHadoopConf(),
+ deletionVectorFilePath,
+ deleteVector.sizeInBytes(),
+ offset);
+ return Arrays.stream(ordinals).iterator();
+ }
+
+ private static int getOffset(DeletionVectorDescriptor deleteVector) {
+ return deleteVector.offset().isDefined() ? (int) deleteVector.offset().get() : 1;
+ }
+
+ @VisibleForTesting
+ long[] parseOrdinalFile(Configuration conf, Path filePath, int size, int offset) {
+ DeletionVectorStore dvStore = new HadoopFileSystemDVStore(conf);
+ RoaringBitmapArray rbm = dvStore.read(filePath, offset, size);
+ return rbm.values();
}
}
diff --git a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaConversionSource.java b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaConversionSource.java
index 97804d5f..554c3f64 100644
--- a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaConversionSource.java
+++ b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaConversionSource.java
@@ -22,11 +22,11 @@ import java.sql.Timestamp;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import lombok.Builder;
import lombok.extern.log4j.Log4j2;
@@ -53,6 +53,7 @@ import org.apache.xtable.model.TableChange;
import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.storage.FileFormat;
import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalDeletionVector;
import org.apache.xtable.model.storage.InternalFilesDiff;
import org.apache.xtable.model.storage.PartitionFileGroup;
import org.apache.xtable.spi.extractor.ConversionSource;
@@ -113,8 +114,8 @@ public class DeltaConversionSource implements ConversionSource<Long> {
// All 3 of the following data structures use data file's absolute path as the key
Map<String, InternalDataFile> addedFiles = new HashMap<>();
Map<String, InternalDataFile> removedFiles = new HashMap<>();
- // Set of data file paths for which deletion vectors exists.
- Set<String> deletionVectors = new HashSet<>();
+ // Map of data file paths for which deletion vectors exists.
+ Map<String, InternalDeletionVector> deletionVectors = new HashMap<>();
for (Action action : actionsForVersion) {
if (action instanceof AddFile) {
@@ -129,10 +130,10 @@ public class DeltaConversionSource implements ConversionSource<Long> {
DeltaPartitionExtractor.getInstance(),
DeltaStatsExtractor.getInstance());
addedFiles.put(dataFile.getPhysicalPath(), dataFile);
- String deleteVectorPath =
- actionsConverter.extractDeletionVectorFile(snapshotAtVersion, (AddFile) action);
- if (deleteVectorPath != null) {
- deletionVectors.add(deleteVectorPath);
+ InternalDeletionVector deletionVector =
+ actionsConverter.extractDeletionVector(snapshotAtVersion, (AddFile) action);
+ if (deletionVector != null) {
+ deletionVectors.put(deletionVector.dataFilePath(), deletionVector);
}
} else if (action instanceof RemoveFile) {
InternalDataFile dataFile =
@@ -151,7 +152,7 @@ public class DeltaConversionSource implements ConversionSource<Long> {
// entry which is replaced by a new entry, AddFile with delete vector information. Since the
// same data file is removed and added, we need to remove it from the added and removed file
// maps which are used to track actual added and removed data files.
- for (String deletionVector : deletionVectors) {
+ for (String deletionVector : deletionVectors.keySet()) {
// validate that a Remove action is also added for the data file
if (removedFiles.containsKey(deletionVector)) {
addedFiles.remove(deletionVector);
@@ -163,11 +164,15 @@ public class DeltaConversionSource implements ConversionSource<Long> {
}
}
+ List<InternalDataFile> allAddedFiles =
+ Stream.concat(addedFiles.values().stream(), deletionVectors.values().stream())
+ .collect(Collectors.toList());
InternalFilesDiff internalFilesDiff =
InternalFilesDiff.builder()
- .filesAdded(addedFiles.values())
+ .filesAdded(allAddedFiles)
.filesRemoved(removedFiles.values())
.build();
+
return TableChange.builder()
.tableAsOfChange(tableAtVersion)
.filesDiff(internalFilesDiff)
diff --git a/xtable-core/src/test/java/org/apache/xtable/TestSparkDeltaTable.java b/xtable-core/src/test/java/org/apache/xtable/TestSparkDeltaTable.java
index ee5b1ccd..909b1b79 100644
--- a/xtable-core/src/test/java/org/apache/xtable/TestSparkDeltaTable.java
+++ b/xtable-core/src/test/java/org/apache/xtable/TestSparkDeltaTable.java
@@ -39,6 +39,7 @@ import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.functions;
import org.apache.spark.sql.delta.DeltaLog;
+import org.apache.spark.sql.delta.actions.AddFile;
import com.google.common.base.Preconditions;
@@ -212,11 +213,15 @@ public class TestSparkDeltaTable implements GenericTable<Row, Object>, Closeable
}
public List<String> getAllActiveFiles() {
- return deltaLog.snapshot().allFiles().collectAsList().stream()
+ return getAllActiveFilesInfo().stream()
.map(addFile -> addSlashToBasePath(basePath) + addFile.path())
.collect(Collectors.toList());
}
+ public List<AddFile> getAllActiveFilesInfo() {
+ return deltaLog.snapshot().allFiles().collectAsList();
+ }
+
private String addSlashToBasePath(String basePath) {
if (basePath.endsWith("/")) {
return basePath;
diff --git a/xtable-core/src/test/java/org/apache/xtable/ValidationTestHelper.java b/xtable-core/src/test/java/org/apache/xtable/ValidationTestHelper.java
index 9e95f279..fc64c513 100644
--- a/xtable-core/src/test/java/org/apache/xtable/ValidationTestHelper.java
+++ b/xtable-core/src/test/java/org/apache/xtable/ValidationTestHelper.java
@@ -30,6 +30,7 @@ import java.util.stream.IntStream;
import org.apache.xtable.model.InternalSnapshot;
import org.apache.xtable.model.TableChange;
import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalDeletionVector;
public class ValidationTestHelper {
@@ -96,7 +97,10 @@ public class ValidationTestHelper {
}
private static Set<String> extractPathsFromDataFile(Set<InternalDataFile> dataFiles) {
- return dataFiles.stream().map(InternalDataFile::getPhysicalPath).collect(Collectors.toSet());
+ return dataFiles.stream()
+ .filter(file -> !(file instanceof InternalDeletionVector))
+ .map(InternalDataFile::getPhysicalPath)
+ .collect(Collectors.toSet());
}
private static void replaceFileScheme(List<String> filePaths) {
diff --git a/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaDeleteVectorConvert.java b/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaDeleteVectorConvert.java
index ed02893e..dcfb5f80 100644
--- a/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaDeleteVectorConvert.java
+++ b/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaDeleteVectorConvert.java
@@ -19,11 +19,16 @@
package org.apache.xtable.delta;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.time.Instant;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
@@ -38,6 +43,7 @@ import org.junit.jupiter.api.io.TempDir;
import org.apache.spark.sql.delta.DeltaLog;
import org.apache.spark.sql.delta.actions.AddFile;
+import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor;
import scala.Option;
@@ -49,6 +55,7 @@ import org.apache.xtable.model.CommitsBacklog;
import org.apache.xtable.model.InstantsForIncrementalSync;
import org.apache.xtable.model.InternalSnapshot;
import org.apache.xtable.model.TableChange;
+import org.apache.xtable.model.storage.InternalDeletionVector;
import org.apache.xtable.model.storage.TableFormat;
public class ITDeltaDeleteVectorConvert {
@@ -56,6 +63,7 @@ public class ITDeltaDeleteVectorConvert {
private static SparkSession sparkSession;
private DeltaConversionSourceProvider conversionSourceProvider;
+ private TestSparkDeltaTable testSparkDeltaTable;
@BeforeAll
public static void setupOnce() {
@@ -91,11 +99,24 @@ public class ITDeltaDeleteVectorConvert {
conversionSourceProvider.init(hadoopConf);
}
+ private static class TableState {
+ Map<String, AddFile> activeFiles;
+ List<Row> rowsToDelete;
+
+ TableState(Map<String, AddFile> activeFiles) {
+ this(activeFiles, Collections.emptyList());
+ }
+
+ TableState(Map<String, AddFile> activeFiles, List<Row> rowsToDelete) {
+ this.activeFiles = activeFiles;
+ this.rowsToDelete = rowsToDelete;
+ }
+ }
+
@Test
public void testInsertsUpsertsAndDeletes() {
String tableName = GenericTable.getTableName();
- TestSparkDeltaTable testSparkDeltaTable =
- new TestSparkDeltaTable(tableName, tempDir, sparkSession, null, false);
+ testSparkDeltaTable = new TestSparkDeltaTable(tableName, tempDir, sparkSession, null, false);
// enable deletion vectors for the test table
testSparkDeltaTable
@@ -105,25 +126,30 @@ public class ITDeltaDeleteVectorConvert {
+ tableName
+ " SET TBLPROPERTIES ('delta.enableDeletionVectors' = true)");
- List<List<String>> allActiveFiles = new ArrayList<>();
+ List<TableState> testTableStates = new ArrayList<>();
List<TableChange> allTableChanges = new ArrayList<>();
List<Row> rows = testSparkDeltaTable.insertRows(50);
Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
- allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+ Map<String, AddFile> tableFiles = collectActiveFilesAfterCommit(testSparkDeltaTable);
+ testTableStates.add(new TableState(tableFiles, Collections.emptyList()));
List<Row> rows1 = testSparkDeltaTable.insertRows(50);
- allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+ tableFiles = collectActiveFilesAfterCommit(testSparkDeltaTable);
+ testTableStates.add(new TableState(tableFiles));
+ validateDeletedRecordCount(testSparkDeltaTable.getDeltaLog(), 0, 0);
assertEquals(100L, testSparkDeltaTable.getNumRows());
- validateDeletedRecordCount(testSparkDeltaTable.getDeltaLog(), allActiveFiles.size() + 1, 0, 0);
// upsert does not create delete vectors
testSparkDeltaTable.upsertRows(rows.subList(0, 20));
- allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+ tableFiles = collectActiveFilesAfterCommit(testSparkDeltaTable);
+ testTableStates.add(new TableState(tableFiles));
+ validateDeletedRecordCount(testSparkDeltaTable.getDeltaLog(), 0, 0);
assertEquals(100L, testSparkDeltaTable.getNumRows());
- validateDeletedRecordCount(testSparkDeltaTable.getDeltaLog(), allActiveFiles.size() + 1, 0, 0);
testSparkDeltaTable.insertRows(50);
- allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+ tableFiles = collectActiveFilesAfterCommit(testSparkDeltaTable);
+ testTableStates.add(new TableState(tableFiles));
+ validateDeletedRecordCount(testSparkDeltaTable.getDeltaLog(), 0, 0);
assertEquals(150L, testSparkDeltaTable.getNumRows());
// delete a few rows with gaps in ids
@@ -133,12 +159,15 @@ public class ITDeltaDeleteVectorConvert {
.collect(Collectors.toList());
rowsToDelete.addAll(rows.subList(35, 45));
testSparkDeltaTable.deleteRows(rowsToDelete);
- allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+ tableFiles = collectActiveFilesAfterCommit(testSparkDeltaTable);
+ testTableStates.add(new TableState(tableFiles, rowsToDelete));
+ validateDeletedRecordCount(testSparkDeltaTable.getDeltaLog(), 2, 15);
assertEquals(135L, testSparkDeltaTable.getNumRows());
- validateDeletedRecordCount(testSparkDeltaTable.getDeltaLog(), allActiveFiles.size() + 1, 2, 15);
testSparkDeltaTable.insertRows(50);
- allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+ tableFiles = collectActiveFilesAfterCommit(testSparkDeltaTable);
+ testTableStates.add(new TableState(tableFiles));
+ validateDeletedRecordCount(testSparkDeltaTable.getDeltaLog(), 2, 15);
assertEquals(185L, testSparkDeltaTable.getNumRows());
// delete a few rows from a file which already has a deletion vector, this should generate a
@@ -146,18 +175,22 @@ public class ITDeltaDeleteVectorConvert {
// This deletion step intentionally deletes the same rows again to test the merge.
rowsToDelete = rows1.subList(5, 15);
testSparkDeltaTable.deleteRows(rowsToDelete);
- allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+ tableFiles = collectActiveFilesAfterCommit(testSparkDeltaTable);
+ testTableStates.add(new TableState(tableFiles, rowsToDelete));
+ validateDeletedRecordCount(testSparkDeltaTable.getDeltaLog(), 2, 22);
assertEquals(178L, testSparkDeltaTable.getNumRows());
- validateDeletedRecordCount(testSparkDeltaTable.getDeltaLog(), allActiveFiles.size() + 1, 2, 22);
testSparkDeltaTable.insertRows(50);
- allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+ tableFiles = collectActiveFilesAfterCommit(testSparkDeltaTable);
+ testTableStates.add(new TableState(tableFiles));
+ validateDeletedRecordCount(testSparkDeltaTable.getDeltaLog(), 2, 22);
assertEquals(228L, testSparkDeltaTable.getNumRows());
+ String tableBasePath = testSparkDeltaTable.getBasePath();
SourceTable tableConfig =
SourceTable.builder()
.name(testSparkDeltaTable.getTableName())
- .basePath(testSparkDeltaTable.getBasePath())
+ .basePath(tableBasePath)
.formatName(TableFormat.DELTA)
.build();
DeltaConversionSource conversionSource =
@@ -165,8 +198,9 @@ public class ITDeltaDeleteVectorConvert {
InternalSnapshot internalSnapshot = conversionSource.getCurrentSnapshot();
// validateDeltaPartitioning(internalSnapshot);
- ValidationTestHelper.validateSnapshot(
- internalSnapshot, allActiveFiles.get(allActiveFiles.size() - 1));
+ List<String> activeDataFilePaths =
+ new ArrayList<>(testTableStates.get(testTableStates.size() - 1).activeFiles.keySet());
+ ValidationTestHelper.validateSnapshot(internalSnapshot, activeDataFilePaths);
// Get changes in incremental format.
InstantsForIncrementalSync instantsForIncrementalSync =
@@ -179,13 +213,126 @@ public class ITDeltaDeleteVectorConvert {
TableChange tableChange = conversionSource.getTableChangeForCommit(version);
allTableChanges.add(tableChange);
}
- ValidationTestHelper.validateTableChanges(allActiveFiles, allTableChanges);
+
+ List<List<String>> allActiveDataFilePaths =
+ testTableStates.stream()
+ .map(s -> s.activeFiles)
+ .map(Map::keySet)
+ .map(ArrayList::new)
+ .collect(Collectors.toList());
+ ValidationTestHelper.validateTableChanges(allActiveDataFilePaths, allTableChanges);
+
+ validateDeletionInfo(testTableStates, allTableChanges);
+ }
+
+ // collects active files in the current snapshot as a map and adds it to the list
+ private Map<String, AddFile> collectActiveFilesAfterCommit(
+ TestSparkDeltaTable testSparkDeltaTable) {
+ Map<String, AddFile> allFiles =
+ testSparkDeltaTable.getAllActiveFilesInfo().stream()
+ .collect(
+ Collectors.toMap(
+ file -> getAddFileAbsolutePath(file, testSparkDeltaTable.getBasePath()),
+ file -> file));
+ return allFiles;
+ }
+
+ private void validateDeletionInfo(
+ List<TableState> testTableStates, List<TableChange> allTableChanges) {
+ if (allTableChanges.isEmpty() && testTableStates.size() <= 1) {
+ return;
+ }
+
+ assertEquals(
+ allTableChanges.size(),
+ testTableStates.size() - 1,
+ "Number of table changes should be equal to number of commits - 1");
+
+ for (int i = 0; i < allTableChanges.size() - 1; i++) {
+ Map<String, AddFile> activeFileAfterCommit = testTableStates.get(i + 1).activeFiles;
+ Map<String, AddFile> activeFileBeforeCommit = testTableStates.get(i).activeFiles;
+
+ Map<String, AddFile> activeFilesWithUpdatedDeleteInfo =
+ activeFileAfterCommit.entrySet().stream()
+ .filter(e -> e.getValue().deletionVector() != null)
+ .filter(
+ entry -> {
+ if (activeFileBeforeCommit.get(entry.getKey()) == null) {
+ return true;
+ }
+ if (activeFileBeforeCommit.get(entry.getKey()).deletionVector() == null) {
+ return true;
+ }
+ DeletionVectorDescriptor deletionVectorDescriptor =
+ activeFileBeforeCommit.get(entry.getKey()).deletionVector();
+ return !deletionVectorDescriptor.equals(entry.getValue().deletionVector());
+ })
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+ if (activeFilesWithUpdatedDeleteInfo.isEmpty()) {
+ continue;
+ }
+
+ // validate all new delete vectors are correctly detected
+ validateDeletionInfoForCommit(
+ testTableStates.get(i + 1), activeFilesWithUpdatedDeleteInfo, allTableChanges.get(i));
+ }
+ }
+
+ private void validateDeletionInfoForCommit(
+ TableState tableState,
+ Map<String, AddFile> activeFilesAfterCommit,
+ TableChange changeDetectedForCommit) {
+ Map<String, InternalDeletionVector> detectedDeleteInfos =
+ changeDetectedForCommit.getFilesDiff().getFilesAdded().stream()
+ .filter(file -> file instanceof InternalDeletionVector)
+ .map(file -> (InternalDeletionVector) file)
+ .collect(Collectors.toMap(InternalDeletionVector::dataFilePath, file -> file));
+
+ Map<String, AddFile> filesWithDeleteVectors =
+ activeFilesAfterCommit.entrySet().stream()
+ .filter(file -> file.getValue().deletionVector() != null)
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+ assertEquals(filesWithDeleteVectors.size(), detectedDeleteInfos.size());
+
+ for (Map.Entry<String, AddFile> fileWithDeleteVector : filesWithDeleteVectors.entrySet()) {
+ InternalDeletionVector deleteInfo = detectedDeleteInfos.get(fileWithDeleteVector.getKey());
+ assertNotNull(deleteInfo);
+ DeletionVectorDescriptor deletionVectorDescriptor =
+ fileWithDeleteVector.getValue().deletionVector();
+ assertEquals(deletionVectorDescriptor.cardinality(), deleteInfo.getRecordCount());
+ assertEquals(deletionVectorDescriptor.sizeInBytes(), deleteInfo.getFileSizeBytes());
+ assertEquals(deletionVectorDescriptor.offset().get(), deleteInfo.offset());
+
+ String deletionFilePath =
+ deletionVectorDescriptor
+ .absolutePath(new org.apache.hadoop.fs.Path(testSparkDeltaTable.getBasePath()))
+ .toString();
+ assertEquals(deletionFilePath, deleteInfo.getPhysicalPath());
+
+ Iterator<Long> iterator = deleteInfo.ordinalsIterator();
+ List<Long> deletes = new ArrayList<>();
+ iterator.forEachRemaining(deletes::add);
+ assertEquals(deletes.size(), deleteInfo.getRecordCount());
+ }
+ }
+
+ private static String getAddFileAbsolutePath(AddFile file, String tableBasePath) {
+ String filePath = file.path();
+ if (filePath.startsWith(tableBasePath)) {
+ return filePath;
+ }
+ return Paths.get(tableBasePath, file.path()).toString();
}
private void validateDeletedRecordCount(
- DeltaLog deltaLog, int version, int deleteVectorFileCount, int deletionRecordCount) {
+ DeltaLog deltaLog, int deleteVectorFileCount, int deletionRecordCount) {
List<AddFile> allFiles =
- deltaLog.getSnapshotAt(version, Option.empty()).allFiles().collectAsList();
+ deltaLog
+ .getSnapshotAt(deltaLog.snapshot().version(), Option.empty())
+ .allFiles()
+ .collectAsList();
List<AddFile> filesWithDeletionVectors =
allFiles.stream().filter(f -> f.deletionVector() != null).collect(Collectors.toList());
diff --git a/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaActionsConverter.java b/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaActionsConverter.java
index e62e9341..117b3fc7 100644
--- a/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaActionsConverter.java
+++ b/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaActionsConverter.java
@@ -18,10 +18,20 @@
package org.apache.xtable.delta;
-import java.net.URISyntaxException;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.UUID;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
@@ -29,39 +39,155 @@ import org.apache.spark.sql.delta.DeltaLog;
import org.apache.spark.sql.delta.Snapshot;
import org.apache.spark.sql.delta.actions.AddFile;
import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor;
+import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray;
+import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArrayFormat;
import scala.Option;
+import org.apache.xtable.model.storage.InternalDeletionVector;
+
class TestDeltaActionsConverter {
+ private final String basePath = "https://container.blob.core.windows.net/tablepath/";
+ private final int size = 372;
+ private final long time = 376;
+ private final boolean dataChange = true;
+ private final String stats = "";
+ private final int cardinality = 42;
+ private final int offset = 634;
+
@Test
- void extractDeletionVector() throws URISyntaxException {
+ void extractMissingDeletionVector() {
DeltaActionsConverter actionsConverter = DeltaActionsConverter.getInstance();
- int size = 123;
- long time = 234L;
- boolean dataChange = true;
- String stats = "";
- String filePath = "https://container.blob.core.windows.net/tablepath/file_path";
+ String filePath = basePath + "file_path";
Snapshot snapshot = Mockito.mock(Snapshot.class);
- DeltaLog deltaLog = Mockito.mock(DeltaLog.class);
DeletionVectorDescriptor deletionVector = null;
AddFile addFileAction =
new AddFile(filePath, null, size, time, dataChange, stats, null, deletionVector);
- Assertions.assertNull(actionsConverter.extractDeletionVectorFile(snapshot, addFileAction));
+ InternalDeletionVector internaldeletionVector =
+ actionsConverter.extractDeletionVector(snapshot, addFileAction);
+ assertNull(internaldeletionVector);
+ }
- deletionVector =
+ @Test
+ void extractDeletionVectorInFileAbsolutePath() {
+ DeltaActionsConverter actionsConverter = spy(DeltaActionsConverter.getInstance());
+
+ String dataFilePath = "data_file";
+ String deleteFilePath = "https://container.blob.core.windows.net/tablepath/delete_file";
+ Snapshot snapshot = Mockito.mock(Snapshot.class);
+
+ DeletionVectorDescriptor deletionVector =
DeletionVectorDescriptor.onDiskWithAbsolutePath(
- filePath, size, 42, Option.empty(), Option.empty());
+ deleteFilePath, size, cardinality, Option.apply(offset), Option.empty());
- addFileAction =
- new AddFile(filePath, null, size, time, dataChange, stats, null, deletionVector);
+ AddFile addFileAction =
+ new AddFile(dataFilePath, null, size, time, dataChange, stats, null, deletionVector);
+
+ Configuration conf = new Configuration();
+ DeltaLog deltaLog = Mockito.mock(DeltaLog.class);
+ when(snapshot.deltaLog()).thenReturn(deltaLog);
+ when(deltaLog.dataPath()).thenReturn(new Path(basePath));
+ when(deltaLog.newDeltaHadoopConf()).thenReturn(conf);
+
+ long[] ordinals = {45, 78, 98};
+ Mockito.doReturn(ordinals)
+ .when(actionsConverter)
+ .parseOrdinalFile(conf, new Path(deleteFilePath), size, offset);
+
+ InternalDeletionVector internaldeletionVector =
+ actionsConverter.extractDeletionVector(snapshot, addFileAction);
+ assertNotNull(internaldeletionVector);
+ assertEquals(basePath + dataFilePath, internaldeletionVector.dataFilePath());
+ assertEquals(deleteFilePath, internaldeletionVector.getPhysicalPath());
+ assertEquals(offset, internaldeletionVector.offset());
+ assertEquals(cardinality, internaldeletionVector.getRecordCount());
+ assertEquals(size, internaldeletionVector.getFileSizeBytes());
+ assertNull(internaldeletionVector.binaryRepresentation());
+
+ Iterator<Long> iterator = internaldeletionVector.ordinalsIterator();
+ Arrays.stream(ordinals).forEach(o -> assertEquals(o, iterator.next()));
+ assertFalse(iterator.hasNext());
+ }
+
+ @Test
+ void extractDeletionVectorInFileRelativePath() {
+ DeltaActionsConverter actionsConverter = spy(DeltaActionsConverter.getInstance());
+
+ String dataFilePath = "data_file";
+ UUID deleteFileId = UUID.randomUUID();
+ String deleteFilePath = basePath + "deletion_vector_" + deleteFileId + ".bin";
+ Snapshot snapshot = Mockito.mock(Snapshot.class);
+
+ DeletionVectorDescriptor deletionVector =
+ DeletionVectorDescriptor.onDiskWithRelativePath(
+ deleteFileId, "", size, cardinality, Option.apply(offset), Option.empty());
+
+ AddFile addFileAction =
+ new AddFile(dataFilePath, null, size, time, dataChange, stats, null, deletionVector);
+
+ Configuration conf = new Configuration();
+ DeltaLog deltaLog = Mockito.mock(DeltaLog.class);
+ when(snapshot.deltaLog()).thenReturn(deltaLog);
+ when(deltaLog.dataPath()).thenReturn(new Path(basePath));
+ when(deltaLog.newDeltaHadoopConf()).thenReturn(conf);
+
+ long[] ordinals = {45, 78, 98};
+ Mockito.doReturn(ordinals)
+ .when(actionsConverter)
+ .parseOrdinalFile(conf, new Path(deleteFilePath), size, offset);
+
+ InternalDeletionVector internaldeletionVector =
+ actionsConverter.extractDeletionVector(snapshot, addFileAction);
+ assertNotNull(internaldeletionVector);
+ assertEquals(basePath + dataFilePath, internaldeletionVector.dataFilePath());
+ assertEquals(deleteFilePath, internaldeletionVector.getPhysicalPath());
+ assertEquals(offset, internaldeletionVector.offset());
+ assertEquals(cardinality, internaldeletionVector.getRecordCount());
+ assertEquals(size, internaldeletionVector.getFileSizeBytes());
+ assertNull(internaldeletionVector.binaryRepresentation());
+
+ Iterator<Long> iterator = internaldeletionVector.ordinalsIterator();
+ Arrays.stream(ordinals).forEach(o -> assertEquals(o, iterator.next()));
+ assertFalse(iterator.hasNext());
+ }
+
+ @Test
+ void extractInlineDeletionVector() {
+ DeltaActionsConverter actionsConverter = spy(DeltaActionsConverter.getInstance());
+
+ String dataFilePath = "data_file";
+ Snapshot snapshot = Mockito.mock(Snapshot.class);
+
+ long[] ordinals = {45, 78, 98};
+ RoaringBitmapArray rbm = new RoaringBitmapArray();
+ Arrays.stream(ordinals).forEach(rbm::add);
+ byte[] bytes = rbm.serializeAsByteArray(RoaringBitmapArrayFormat.Portable());
+
+ DeletionVectorDescriptor deletionVector =
+ DeletionVectorDescriptor.inlineInLog(bytes, cardinality);
+
+ AddFile addFileAction =
+ new AddFile(dataFilePath, null, size, time, dataChange, stats, null, deletionVector);
+
+ DeltaLog deltaLog = Mockito.mock(DeltaLog.class);
+ when(snapshot.deltaLog()).thenReturn(deltaLog);
+ when(deltaLog.dataPath()).thenReturn(new Path(basePath));
+
+ InternalDeletionVector internaldeletionVector =
+ actionsConverter.extractDeletionVector(snapshot, addFileAction);
+ assertNotNull(internaldeletionVector);
+ assertEquals(basePath + dataFilePath, internaldeletionVector.dataFilePath());
+ assertArrayEquals(bytes, internaldeletionVector.binaryRepresentation());
+ assertEquals(cardinality, internaldeletionVector.getRecordCount());
+ assertEquals(bytes.length, internaldeletionVector.getFileSizeBytes());
+ assertEquals("", internaldeletionVector.getPhysicalPath());
+ assertEquals(0, internaldeletionVector.offset());
- Mockito.when(snapshot.deltaLog()).thenReturn(deltaLog);
- Mockito.when(deltaLog.dataPath())
- .thenReturn(new Path("https://container.blob.core.windows.net/tablepath"));
- Assertions.assertEquals(
- filePath, actionsConverter.extractDeletionVectorFile(snapshot, addFileAction));
+ Iterator<Long> iterator = internaldeletionVector.ordinalsIterator();
+ Arrays.stream(ordinals).forEach(o -> assertEquals(o, iterator.next()));
+ assertFalse(iterator.hasNext());
}
}