You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2026/04/10 08:00:54 UTC
(ozone) branch master updated: HDDS-14938. Implement Iceberg RewriteTablePath action (#10053)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e8fac639e21 HDDS-14938. Implement Iceberg RewriteTablePath action (#10053)
e8fac639e21 is described below
commit e8fac639e2151af8bda9ae8928c69fbd2a774a12
Author: sreejasahithi <11...@users.noreply.github.com>
AuthorDate: Fri Apr 10 13:30:48 2026 +0530
HDDS-14938. Implement Iceberg RewriteTablePath action (#10053)
---
hadoop-ozone/dist/pom.xml | 13 ++
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 5 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 6 +
.../iceberg/dev-support/findbugsExcludeFile.xml | 16 ++
hadoop-ozone/iceberg/pom.xml | 99 ++++++++++
.../ozone/iceberg/RewriteTablePathOzoneAction.java | 219 +++++++++++++++++++++
hadoop-ozone/pom.xml | 9 +
pom.xml | 13 ++
8 files changed, 380 insertions(+)
diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml
index eb7f2bc2430..cdbd5a48735 100644
--- a/hadoop-ozone/dist/pom.xml
+++ b/hadoop-ozone/dist/pom.xml
@@ -255,6 +255,19 @@
</plugins>
</build>
<profiles>
+ <profile>
+ <id>jdk11-with-iceberg</id>
+ <activation>
+ <jdk>[11,)</jdk>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>ozone-iceberg</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ </dependencies>
+ </profile>
<profile>
<id>build-with-ozonefs</id>
<activation>
diff --git a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
index a87c952ca76..9670a99ed35 100644
--- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
+++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
@@ -280,6 +280,7 @@ Apache License 2.0
com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider
com.fasterxml.jackson.module:jackson-module-jaxb-annotations
com.fasterxml.woodstox:woodstox-core
+ com.github.ben-manes.caffeine:caffeine
com.github.jnr:jnr-a64asm
com.github.jnr:jnr-constants
com.github.jnr:jnr-ffi
@@ -366,6 +367,7 @@ Apache License 2.0
log4j:apache-log4j-extras
net.java.dev.jna:jna
net.java.dev.jna:jna-platform
+ org.apache.avro:avro
org.apache.commons:commons-compress
org.apache.commons:commons-configuration2
org.apache.commons:commons-collections4
@@ -384,6 +386,9 @@ Apache License 2.0
org.apache.hadoop:hadoop-shaded-guava
org.apache.hadoop:hadoop-shaded-protobuf_3_25
org.apache.httpcomponents:httpcore
+ org.apache.iceberg:iceberg-api
+ org.apache.iceberg:iceberg-common
+ org.apache.iceberg:iceberg-core
org.apache.kerby:kerb-admin
org.apache.kerby:kerb-client
org.apache.kerby:kerb-common
diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt
index 0d142d08c95..bfe9a8ecf35 100644
--- a/hadoop-ozone/dist/src/main/license/jar-report.txt
+++ b/hadoop-ozone/dist/src/main/license/jar-report.txt
@@ -11,6 +11,7 @@ share/ozone/lib/asm.jar
share/ozone/lib/asm-tree.jar
share/ozone/lib/asm-util.jar
share/ozone/lib/aspectjrt.jar
+share/ozone/lib/avro.jar
share/ozone/lib/aws-java-sdk-core.jar
share/ozone/lib/aws-java-sdk-kms.jar
share/ozone/lib/aws-java-sdk-s3.jar
@@ -18,6 +19,7 @@ share/ozone/lib/bcpkix-jdk18on.jar
share/ozone/lib/bcprov-jdk18on.jar
share/ozone/lib/bcutil-jdk18on.jar
share/ozone/lib/bonecp.RELEASE.jar
+share/ozone/lib/caffeine.jar
share/ozone/lib/cdi-api.SP1.jar
share/ozone/lib/commons-beanutils.jar
share/ozone/lib/commons-cli.jar
@@ -84,6 +86,9 @@ share/ozone/lib/hk2-utils.jar
share/ozone/lib/hppc.jar
share/ozone/lib/httpclient.jar
share/ozone/lib/httpcore.jar
+share/ozone/lib/iceberg-api.jar
+share/ozone/lib/iceberg-common.jar
+share/ozone/lib/iceberg-core.jar
share/ozone/lib/istack-commons-runtime.jar
share/ozone/lib/j2objc-annotations.jar
share/ozone/lib/jackson-annotations.jar
@@ -223,6 +228,7 @@ share/ozone/lib/ozone-filesystem-hadoop3.jar
share/ozone/lib/ozone-filesystem.jar
share/ozone/lib/ozone-freon.jar
share/ozone/lib/ozone-httpfsgateway.jar
+share/ozone/lib/ozone-iceberg.jar
share/ozone/lib/ozone-insight.jar
share/ozone/lib/ozone-interface-client.jar
share/ozone/lib/ozone-interface-storage.jar
diff --git a/hadoop-ozone/iceberg/dev-support/findbugsExcludeFile.xml b/hadoop-ozone/iceberg/dev-support/findbugsExcludeFile.xml
new file mode 100644
index 00000000000..ee5ed59808b
--- /dev/null
+++ b/hadoop-ozone/iceberg/dev-support/findbugsExcludeFile.xml
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<FindBugsFilter>
+</FindBugsFilter>
diff --git a/hadoop-ozone/iceberg/pom.xml b/hadoop-ozone/iceberg/pom.xml
new file mode 100644
index 00000000000..c56fabb8a27
--- /dev/null
+++ b/hadoop-ozone/iceberg/pom.xml
@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>ozone</artifactId>
+ <version>2.2.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>ozone-iceberg</artifactId>
+ <version>2.2.0-SNAPSHOT</version>
+ <packaging>jar</packaging>
+ <name>Apache Ozone Iceberg Integration</name>
+ <description>Apache Ozone Iceberg Integration</description>
+
+ <properties>
+ <classpath.skip>false</classpath.skip>
+ <!-- Iceberg 1.10+ API is Java 11 bytecode, override parent Java 8 default. -->
+ <maven.compiler.release>11</maven.compiler.release>
+ </properties>
+
+ <dependencies>
+
+ <!-- Iceberg dependencies -->
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-api</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-bundled-guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-core</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents.client5</groupId>
+ <artifactId>httpclient5</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents.core5</groupId>
+ <artifactId>httpcore5</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents.core5</groupId>
+ <artifactId>httpcore5-h2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-bundled-guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.checkerframework</groupId>
+ <artifactId>checker-qual</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.roaringbitmap</groupId>
+ <artifactId>RoaringBitmap</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <proc>none</proc>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
+ <configuration>
+ <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
+ <fork>true</fork>
+ <maxHeap>2048</maxHeap>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathOzoneAction.java b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathOzoneAction.java
new file mode 100644
index 00000000000..176bcbf2353
--- /dev/null
+++ b/hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathOzoneAction.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.ozone.iceberg;
+
+import java.util.Objects;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.RewriteTablePathUtil;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableMetadata.MetadataLogEntry;
+import org.apache.iceberg.actions.ImmutableRewriteTablePath;
+import org.apache.iceberg.actions.RewriteTablePath;
+
+public class RewriteTablePathOzoneAction implements RewriteTablePath {
+
+ private static final String RESULT_LOCATION = "file-list";
+
+ private String sourcePrefix;
+ private String targetPrefix;
+ private String startVersionName;
+ private String endVersionName;
+ private String stagingDir;
+ private int parallelism;
+
+ private final Table table;
+ private ExecutorService executorService;
+
+ public RewriteTablePathOzoneAction(Table table) {
+ this.table = table;
+ this.parallelism = Runtime.getRuntime().availableProcessors();
+ }
+
+ public RewriteTablePathOzoneAction(Table table, int parallelism) {
+ this.table = table;
+ this.parallelism = parallelism;
+ }
+
+ @Override
+ public RewriteTablePath rewriteLocationPrefix(String sPrefix, String tPrefix) {
+ checkNonNullNonEmpty(sPrefix, "Source prefix");
+ checkNonNullNonEmpty(tPrefix, "Target prefix");
+ this.sourcePrefix = sPrefix;
+ this.targetPrefix = tPrefix;
+ return this;
+ }
+
+ @Override
+ public RewriteTablePath startVersion(String sVersion) {
+ checkNonNullNonEmpty(sVersion, "Start version");
+ this.startVersionName = sVersion;
+ return this;
+ }
+
+ @Override
+ public RewriteTablePath endVersion(String eVersion) {
+ checkNonNullNonEmpty(eVersion, "End version");
+ this.endVersionName = eVersion;
+ return this;
+ }
+
+ @Override
+ public RewriteTablePath stagingLocation(String stagingLocation) {
+ checkNonNullNonEmpty(stagingLocation, "Staging location");
+ this.stagingDir = stagingLocation;
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ validateInputs();
+ this.executorService = Executors.newFixedThreadPool(parallelism);
+ try {
+ return doExecute();
+ } finally {
+ executorService.shutdown();
+ try {
+ if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) {
+ executorService.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ executorService.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ private Result doExecute() {
+ String resultLocation = rebuildMetadata();
+ return ImmutableRewriteTablePath.Result.builder()
+ .stagingLocation(stagingDir)
+ .fileListLocation(resultLocation)
+ .latestVersion(RewriteTablePathUtil.fileName(endVersionName))
+ .build();
+ }
+
+ private void validateInputs() {
+ if (sourcePrefix.equals(targetPrefix)) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Source prefix cannot be the same as target prefix (%s)", sourcePrefix));
+ }
+
+ validateAndSetEndVersion();
+ validateAndSetStartVersion();
+
+ if (stagingDir == null) {
+ stagingDir =
+ getMetadataLocation(table)
+ + "copy-table-staging-"
+ + UUID.randomUUID()
+ + RewriteTablePathUtil.FILE_SEPARATOR;
+ } else {
+ stagingDir = RewriteTablePathUtil.maybeAppendFileSeparator(stagingDir);
+ }
+ }
+
+ private void validateAndSetEndVersion() {
+ TableMetadata tableMetadata = ((HasTableOperations) table).operations().current();
+
+ if (endVersionName == null) {
+ Objects.requireNonNull(
+ tableMetadata.metadataFileLocation(), "Metadata file location should not be null");
+ this.endVersionName = tableMetadata.metadataFileLocation();
+ } else {
+ this.endVersionName = validateVersion(tableMetadata, endVersionName);
+ }
+ }
+
+ private void validateAndSetStartVersion() {
+ TableMetadata tableMetadata = ((HasTableOperations) table).operations().current();
+
+ if (startVersionName != null) {
+ this.startVersionName = validateVersion(tableMetadata, startVersionName);
+ }
+ }
+
+ private String validateVersion(TableMetadata tableMetadata, String versionFileName) {
+ String versionFile = null;
+ if (versionInFilePath(tableMetadata.metadataFileLocation(), versionFileName)) {
+ versionFile = tableMetadata.metadataFileLocation();
+ }
+
+ for (MetadataLogEntry log : tableMetadata.previousFiles()) {
+ if (versionInFilePath(log.file(), versionFileName)) {
+ versionFile = log.file();
+ }
+ }
+
+ if (versionFile == null) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Cannot find provided version file %s in metadata log.", versionFileName));
+ }
+ if (!fileExist(versionFile)) {
+ throw new IllegalArgumentException(
+ String.format("Version file %s does not exist.", versionFile));
+ }
+ return versionFile;
+ }
+
+ private boolean versionInFilePath(String path, String version) {
+ return RewriteTablePathUtil.fileName(path).equals(version);
+ }
+
+ private String rebuildMetadata() {
+ //TODO need to implement rewrite of metadata files , manifest list , manifest files and position delete files.
+ return null;
+ }
+
+ private boolean fileExist(String path) {
+ if (path == null || path.trim().isEmpty()) {
+ return false;
+ }
+ return table.io().newInputFile(path).exists();
+ }
+
+ private String getMetadataLocation(Table tbl) {
+ String currentMetadataPath =
+ ((HasTableOperations) tbl).operations().current().metadataFileLocation();
+ int lastIndex = currentMetadataPath.lastIndexOf(RewriteTablePathUtil.FILE_SEPARATOR);
+ String metadataDir = "";
+ if (lastIndex != -1) {
+ metadataDir = currentMetadataPath.substring(0, lastIndex + 1);
+ }
+
+ if (metadataDir.isEmpty()) {
+ throw new IllegalArgumentException("Failed to get the metadata file root directory");
+ }
+ return metadataDir;
+ }
+
+ private static void checkNonNullNonEmpty(String value, String name) {
+ Objects.requireNonNull(value, () -> name + " is null");
+ if (value.trim().isEmpty()) {
+ throw new IllegalArgumentException(name + " is empty");
+ }
+ }
+}
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index e0ab0556441..5118acb5f9e 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -106,6 +106,15 @@
</build>
<profiles>
+ <profile>
+ <id>jdk11-with-iceberg</id>
+ <activation>
+ <jdk>[11,)</jdk>
+ </activation>
+ <modules>
+ <module>iceberg</module>
+ </modules>
+ </profile>
<profile>
<id>build-with-ozonefs</id>
<activation>
diff --git a/pom.xml b/pom.xml
index 7a979814a7c..81fbd721993 100644
--- a/pom.xml
+++ b/pom.xml
@@ -92,6 +92,7 @@
<hk2.version>2.6.1</hk2.version>
<httpclient.version>4.5.14</httpclient.version>
<httpcore.version>4.4.16</httpcore.version>
+ <iceberg.version>1.10.1</iceberg.version>
<io.grpc.version>1.77.1</io.grpc.version>
<jackson2-bom.version>2.21.2</jackson2-bom.version>
<jackson2.version>2.21.2</jackson2.version>
@@ -262,6 +263,13 @@
<type>pom</type>
<scope>import</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-bom</artifactId>
+ <version>${iceberg.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-bom</artifactId>
@@ -1316,6 +1324,11 @@
<artifactId>ozone-httpfsgateway</artifactId>
<version>${ozone.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>ozone-iceberg</artifactId>
+ <version>${ozone.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>ozone-insight</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org