You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@amoro.apache.org by ji...@apache.org on 2024/11/22 09:25:41 UTC
(amoro) branch master updated: [AMORO-3228] Remove the format suffix in judgment condition (#3338)
This is an automated email from the ASF dual-hosted git repository.
jinsongzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new c12c240e6 [AMORO-3228] Remove the format suffix in judgment condition (#3338)
c12c240e6 is described below
commit c12c240e6e5b508962d45fd36675853c4df5062c
Author: Qishang Zhong <zh...@gmail.com>
AuthorDate: Fri Nov 22 17:25:37 2024 +0800
[AMORO-3228] Remove the format suffix in judgment condition (#3338)
* [AMORO-3228] Fix the problem of file name with suffix in judgment condition
* Extract method
* Fix comment
---
.../io/writer/IcebergFanoutPosDeleteWriter.java | 6 +----
.../java/org/apache/amoro/utils/TableFileUtil.java | 26 +++++++++++++++++++---
.../amoro/io/TestIcebergFanoutPosDeleteWriter.java | 4 ++++
3 files changed, 28 insertions(+), 8 deletions(-)
diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/io/writer/IcebergFanoutPosDeleteWriter.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/io/writer/IcebergFanoutPosDeleteWriter.java
index ea3b7ef89..b6f33ac43 100644
--- a/amoro-format-iceberg/src/main/java/org/apache/amoro/io/writer/IcebergFanoutPosDeleteWriter.java
+++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/io/writer/IcebergFanoutPosDeleteWriter.java
@@ -149,11 +149,7 @@ public class IcebergFanoutPosDeleteWriter<T>
return;
}
posDeletes.sort(Comparator.comparingLong(PosRow::pos));
- String fileName = TableFileUtil.getFileName(filePath.get().toString());
- FileFormat fileFormat = FileFormat.fromFileName(fileName);
- if (fileFormat != null) {
- fileName = fileName.substring(0, fileName.length() - fileFormat.name().length() - 1);
- }
+ String fileName = TableFileUtil.getFileNameWithoutExt(filePath.get().toString());
String fileDir = TableFileUtil.getFileDir(filePath.get().toString());
String deleteFilePath =
format.addExtension(
diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/TableFileUtil.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/TableFileUtil.java
index e140c79eb..ea972067b 100644
--- a/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/TableFileUtil.java
+++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/TableFileUtil.java
@@ -20,6 +20,7 @@ package org.apache.amoro.utils;
import org.apache.amoro.io.AuthenticatedFileIO;
import org.apache.hadoop.fs.Path;
+import org.apache.iceberg.FileFormat;
import org.apache.iceberg.io.BulkDeletionFailureException;
import org.apache.iceberg.util.Tasks;
import org.slf4j.Logger;
@@ -36,16 +37,33 @@ public class TableFileUtil {
private static final String POS_DELETE_FILE_IDENTIFIER = "delete";
/**
- * Parse file name form file path
+ * Parse file name from file path.
*
* @param filePath file path
- * @return file name parsed from file path
+ * @return file name parsed from file path, e.g. data-1.parquet.
*/
public static String getFileName(String filePath) {
int lastSlash = filePath.lastIndexOf('/');
return filePath.substring(lastSlash + 1);
}
+ /**
+ * Parse file name without ext from file path.
+ *
+ * @param filePath file path
+ * @return file name without ext parsed from file path, e.g. data-1.
+ */
+ public static String getFileNameWithoutExt(String filePath) {
+ String fileName = getFileName(filePath);
+
+ FileFormat fileFormat = FileFormat.fromFileName(fileName);
+ if (fileFormat != null) {
+ return fileName.substring(0, fileName.length() - fileFormat.name().length() - 1);
+ }
+
+ return fileName;
+ }
+
/**
* Parse file directory path from file path
*
@@ -200,6 +218,8 @@ public class TableFileUtil {
public static boolean isOptimizingPosDeleteFile(String dataFilePath, String posDeleteFilePath) {
return getFileName(posDeleteFilePath)
- .startsWith(String.format("%s-%s", getFileName(dataFilePath), POS_DELETE_FILE_IDENTIFIER));
+ .startsWith(
+ String.format(
+ "%s-%s", getFileNameWithoutExt(dataFilePath), POS_DELETE_FILE_IDENTIFIER));
}
}
diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergFanoutPosDeleteWriter.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergFanoutPosDeleteWriter.java
index ff4185ab0..b8667ce22 100644
--- a/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergFanoutPosDeleteWriter.java
+++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/io/TestIcebergFanoutPosDeleteWriter.java
@@ -127,6 +127,8 @@ public class TestIcebergFanoutPosDeleteWriter extends TableTestBase {
dataDir, fileFormat.addExtension("data-1-delete-suffix")))
.toString());
Assert.assertNotNull(deleteFile1);
+ Assert.assertTrue(
+ TableFileUtil.isOptimizingPosDeleteFile(dataFile1Path, deleteFile1.path().toString()));
Assert.assertEquals(3, deleteFile1.recordCount());
// Check whether the path-pos pairs are sorted as expected.
Schema pathPosSchema = DeleteSchemaUtil.pathPosSchema();
@@ -147,6 +149,8 @@ public class TestIcebergFanoutPosDeleteWriter extends TableTestBase {
dataDir, fileFormat.addExtension("data-2-delete-suffix")))
.toString());
Assert.assertNotNull(deleteFile2);
+ Assert.assertTrue(
+ TableFileUtil.isOptimizingPosDeleteFile(dataFile2Path, deleteFile2.path().toString()));
Assert.assertEquals(
new Path(
TableFileUtil.getNewFilePath(