You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2026/07/08 05:23:53 UTC
(hudi) branch master updated: feat: add FileNameParser for file name decoding (#19198)
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new c56cca6187b1 feat: add FileNameParser for file name decoding (#19198)
c56cca6187b1 is described below
commit c56cca6187b1ad399bc42e987dae90baacf4c55d
Author: Danny Chan <yu...@gmail.com>
AuthorDate: Wed Jul 8 13:23:44 2026 +0800
feat: add FileNameParser for file name decoding (#19198)
---
.../cli/commands/TestFileSystemViewCommand.java | 8 +-
.../index/bucket/ConsistentBucketIndexUtils.java | 3 +-
.../org/apache/hudi/HoodieTestCommitGenerator.java | 2 +-
.../estimator/TestAverageRecordSizeEstimator.java | 2 +-
.../index/expression/TestExpressionIndexer.java | 8 +-
.../table/action/TestFileSliceMetricUtils.java | 2 +-
.../table/action/rollback/TestRollbackHelper.java | 8 +-
.../RDDSimpleBucketBulkInsertPartitioner.java | 4 +-
.../SparkFileFormatInternalRowReaderContext.scala | 5 +-
...parkConsistentBucketClusteringPlanStrategy.java | 2 +-
.../TestSparkSizeBasedClusteringPlanStrategy.java | 2 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 212 ++++----------
.../org/apache/hudi/common/fs/FileNameParser.java | 307 +++++++++++++++++++++
.../apache/hudi/common/model/HoodieBaseFile.java | 38 +--
.../apache/hudi/common/model/HoodieLogFile.java | 54 ++--
.../hudi/common/table/TableSchemaResolver.java | 8 +-
.../table/log/BaseHoodieLogRecordReader.java | 6 +-
.../hudi/common/table/log/HoodieLogFormat.java | 4 +-
.../common/table/log/HoodieLogFormatReader.java | 2 +-
.../common/table/log/NativeLogFooterMetadata.java | 12 +-
.../apache/hudi/common/model/TestFileSlice.java | 8 +-
.../hudi/common/model/TestHoodieBaseFile.java | 42 +++
.../hudi/common/model/TestHoodieLogFile.java | 5 +-
.../serialization/TestDefaultSerializer.java | 2 +-
.../TestHoodieFileGroupSerializer.java | 6 +-
.../TestHoodieFileSliceSerializer.java | 24 +-
.../view/TestHoodieFileGroupSizeEstimator.java | 4 +-
.../view/TestPriorityBasedFileSystemView.java | 6 +-
.../hudi/common/testutils/FileCreateUtils.java | 4 +-
.../hudi/common/testutils/FileCreateUtilsBase.java | 2 +-
.../common/testutils/FileCreateUtilsLegacy.java | 2 +-
.../hudi/common/testutils/HoodieTestUtils.java | 2 +-
.../table/format/FlinkRowDataReaderContext.java | 5 +-
.../apache/hudi/table/format/cdc/CdcIterators.java | 12 +-
.../org/apache/hudi/common/fs/TestFSUtils.java | 115 ++++++--
.../hudi/common/model/TestHoodieFileGroup.java | 6 +-
.../hudi/common/table/TestTableSchemaResolver.java | 4 +-
.../table/view/TestHoodieTableFileSystemView.java | 86 +++---
.../table/view/TestIncrementalFSViewSync.java | 2 +-
.../hudi/common/testutils/HoodieTestTable.java | 2 +-
.../hudi/common/util/TestCompactionUtils.java | 16 +-
.../org/apache/hudi/cdc/CDCFileGroupIterator.scala | 2 +-
...dieSparkMergeOnReadTableInsertUpdateDelete.java | 3 +-
.../TestHoodieMetadataTableValidator.java | 8 +-
44 files changed, 697 insertions(+), 360 deletions(-)
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java
index 9d786704ccb6..1eaa08959ac3 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java
@@ -103,11 +103,11 @@ public class TestFileSystemViewCommand extends CLIFunctionalTestHarness {
Files.createFile(Paths.get(nonpartitionedTablePath, FSUtils
.makeBaseFileName(commitTime1, testWriteToken, fileId1, BASE_FILE_EXTENSION)));
Files.createFile(Paths.get(nonpartitionedTablePath, FSUtils
- .makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, commitTime1, 0, testWriteToken)));
+ .makeInlineLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, commitTime1, 0, testWriteToken)));
Files.createFile(Paths.get(nonpartitionedTablePath, FSUtils
.makeBaseFileName(commitTime2, testWriteToken, fileId1, BASE_FILE_EXTENSION)));
Files.createFile(Paths.get(nonpartitionedTablePath, FSUtils
- .makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, commitTime2, 0, testWriteToken)));
+ .makeInlineLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, commitTime2, 0, testWriteToken)));
// Write commit files
Files.createFile(Paths.get(nonpartitionedTablePath, ".hoodie", commitTime1 + ".commit"));
@@ -146,11 +146,11 @@ public class TestFileSystemViewCommand extends CLIFunctionalTestHarness {
Files.createFile(Paths.get(fullPartitionPath, FSUtils
.makeBaseFileName(commitTime1, testWriteToken, fileId1, BASE_FILE_EXTENSION)));
Files.createFile(Paths.get(fullPartitionPath, FSUtils
- .makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, commitTime1, 0, testWriteToken)));
+ .makeInlineLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, commitTime1, 0, testWriteToken)));
Files.createFile(Paths.get(fullPartitionPath, FSUtils
.makeBaseFileName(commitTime2, testWriteToken, fileId1, BASE_FILE_EXTENSION)));
Files.createFile(Paths.get(fullPartitionPath, FSUtils
- .makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, commitTime2, 0, testWriteToken)));
+ .makeInlineLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, commitTime2, 0, testWriteToken)));
// Write commit files
Files.createFile(Paths.get(partitionedTablePath, ".hoodie", commitTime1 + ".commit"));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
index 16e8a75c01d6..1d41a3a85ee8 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
@@ -19,6 +19,7 @@
package org.apache.hudi.index.bucket;
import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.fs.FileNameParser;
import org.apache.hudi.common.model.ConsistentHashingNode;
import org.apache.hudi.common.model.HoodieConsistentHashingMetadata;
import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -296,7 +297,7 @@ public class ConsistentBucketIndexUtils {
.stream()
.anyMatch(node -> node.getFileIdPrefix().equals(hoodieBaseFile));
if (table.getBaseFileOnlyView().getLatestBaseFiles(partition)
- .map(fileIdPrefix -> FSUtils.getFileIdPfxFromFileId(fileIdPrefix.getFileId())).anyMatch(hoodieFileGroupIdPredicate)) {
+ .map(fileIdPrefix -> FileNameParser.getFileIdPfxFromFileId(fileIdPrefix.getFileId())).anyMatch(hoodieFileGroupIdPredicate)) {
try {
createCommitMarker(table, metaFile.getPath(), metadataPath);
return true;
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/HoodieTestCommitGenerator.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/HoodieTestCommitGenerator.java
index 467a23f5fe80..6ae896ad95ee 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/HoodieTestCommitGenerator.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/HoodieTestCommitGenerator.java
@@ -116,7 +116,7 @@ public class HoodieTestCommitGenerator {
}
public static String getLogFilename(String instantTime, String fileId) {
- return FSUtils.makeLogFileName(
+ return FSUtils.makeInlineLogFileName(
fileId, HoodieFileFormat.HOODIE_LOG.getFileExtension(), instantTime, 1, LOG_FILE_WRITE_TOKEN);
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/execution/estimator/TestAverageRecordSizeEstimator.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/execution/estimator/TestAverageRecordSizeEstimator.java
index 5ef69382f958..762f20df61d0 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/execution/estimator/TestAverageRecordSizeEstimator.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/execution/estimator/TestAverageRecordSizeEstimator.java
@@ -150,7 +150,7 @@ public class TestAverageRecordSizeEstimator {
String fileName = UUID.randomUUID().toString();
String fullFileName = FSUtils.makeBaseFileName(instantTime, TEST_WRITE_TOKEN, fileName, BASE_FILE_EXTENSION);
assertEquals(instantTime, FSUtils.getCommitTime(fullFileName));
- return FSUtils.makeLogFileName(fileName, HOODIE_LOG.getFileExtension(), instantTime, 1, TEST_WRITE_TOKEN);
+ return FSUtils.makeInlineLogFileName(fileName, HOODIE_LOG.getFileExtension(), instantTime, 1, TEST_WRITE_TOKEN);
}
private static Stream<Arguments> testCases() {
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/index/expression/TestExpressionIndexer.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/index/expression/TestExpressionIndexer.java
index 25c7c6a18e1f..d3237d488d23 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/index/expression/TestExpressionIndexer.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/index/expression/TestExpressionIndexer.java
@@ -9,6 +9,7 @@ import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.data.HoodieData;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.engine.HoodieLocalEngineContext;
+import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieIndexDefinition;
@@ -33,6 +34,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
+import java.util.UUID;
import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -87,8 +89,10 @@ class TestExpressionIndexer {
1);
when(engineIndexSupport.generateExpressionIndexRecords(any(), any(), any(), anyInt(), any(), any(), any(), any())).thenReturn(records);
- FileSlice fileSlice = new FileSlice("p1", "001", "f1");
- fileSlice.setBaseFile(new HoodieBaseFile("file:///tmp/p1/f1.parquet"));
+ String fileId = UUID.randomUUID().toString();
+ FileSlice fileSlice = new FileSlice("p1", "001", fileId);
+ String baseFileName = FSUtils.makeBaseFileName("001", "1-0-1", fileId, ".parquet");
+ fileSlice.setBaseFile(new HoodieBaseFile("file:///tmp/p1/" + baseFileName));
List<FileSliceAndPartition> fileSlices = Collections.singletonList(FileSliceAndPartition.of("p1", fileSlice));
try (MockedStatic<HoodieTableMetadataUtil> mockedUtil = mockStatic(HoodieTableMetadataUtil.class)) {
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/TestFileSliceMetricUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/TestFileSliceMetricUtils.java
index eaddde56a0ad..8cc87486e5cb 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/TestFileSliceMetricUtils.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/TestFileSliceMetricUtils.java
@@ -87,7 +87,7 @@ public class TestFileSliceMetricUtils {
}
private FileSlice buildFileSlice(long baseFileLen, List<Long> logFileLens) {
- final String baseFilePath = ".b5068208-e1a4-11e6-bf01-fe55135034f3_20170101134598.log.1";
+ final String baseFilePath = "b5068208-e1a4-11e6-bf01-fe55135034f3_1-0-1_20170101134598.parquet";
FileSlice slice = new FileSlice("partition_0",
InProcessTimeGenerator.createNewInstantTime(),
UUID.randomUUID().toString());
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackHelper.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackHelper.java
index 1f0f801d89b7..b62a2253964d 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackHelper.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackHelper.java
@@ -704,10 +704,10 @@ class TestRollbackHelper extends HoodieRollbackTestBase {
// HoodieLocalEngineContext uses LocalTaskContextSupplier which returns 0/0/0 -> token "0-0-0".
String rollbackWriteToken = FSUtils.makeWriteToken(0, 0, 0);
StoragePath rollbackLogPath1 = new StoragePath(new StoragePath(basePath, ctx.partition2),
- FSUtils.makeLogFileName(ctx.logFileId1, HoodieLogFile.DELTA_EXTENSION,
+ FSUtils.makeInlineLogFileName(ctx.logFileId1, HoodieLogFile.DELTA_EXTENSION,
ctx.baseInstantTimeOfLogFiles, 2, rollbackWriteToken));
StoragePath rollbackLogPath2 = new StoragePath(new StoragePath(basePath, ctx.partition2),
- FSUtils.makeLogFileName(ctx.logFileId2, HoodieLogFile.DELTA_EXTENSION,
+ FSUtils.makeInlineLogFileName(ctx.logFileId2, HoodieLogFile.DELTA_EXTENSION,
ctx.baseInstantTimeOfLogFiles, ROLLBACK_LOG_VERSION, rollbackWriteToken));
List<Pair<String, HoodieRollbackStat>> expected = buildExpectedBaseFileStats(ctx);
@@ -742,7 +742,7 @@ class TestRollbackHelper extends HoodieRollbackTestBase {
String rollbackWriteToken = FSUtils.makeWriteToken(0, 0, 0);
StoragePath rollbackLogPath = new StoragePath(new StoragePath(basePath, ctx.partition),
- FSUtils.makeLogFileName(ctx.logFileId, HoodieLogFile.DELTA_EXTENSION,
+ FSUtils.makeInlineLogFileName(ctx.logFileId, HoodieLogFile.DELTA_EXTENSION,
ctx.baseInstantTimeOfLogFiles, ROLLBACK_LOG_VERSION, rollbackWriteToken));
List<Pair<String, HoodieRollbackStat>> expected = new ArrayList<>();
@@ -810,7 +810,7 @@ class TestRollbackHelper extends HoodieRollbackTestBase {
// log file (the WriterBuilder rediscovers the existing log when we don't explicitly bump the
// version), so it carries the existing log file's write token.
StoragePath rollbackLogPath = new StoragePath(partitionStoragePath,
- FSUtils.makeLogFileName(ctx.logFileId, HoodieLogFile.DELTA_EXTENSION,
+ FSUtils.makeInlineLogFileName(ctx.logFileId, HoodieLogFile.DELTA_EXTENSION,
ctx.baseInstantTimeOfLogFiles, ctx.logVersionCount, HoodieLogFormat.UNKNOWN_WRITE_TOKEN));
List<Pair<String, HoodieRollbackStat>> expected = Collections.singletonList(
Pair.of(ctx.partition,
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDSimpleBucketBulkInsertPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDSimpleBucketBulkInsertPartitioner.java
index ff4e4a4850da..c83f371442e8 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDSimpleBucketBulkInsertPartitioner.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDSimpleBucketBulkInsertPartitioner.java
@@ -18,7 +18,7 @@
package org.apache.hudi.execution.bulkinsert;
-import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.fs.FileNameParser;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordLocation;
@@ -91,7 +91,7 @@ public class RDDSimpleBucketBulkInsertPartitioner<T extends HoodieRecordPayload>
// Load an existing index
locationMap.forEach((k, v) -> {
- String prefix = FSUtils.getFileIdPfxFromFileId(v.getFileId());
+ String prefix = FileNameParser.getFileIdPfxFromFileId(v.getFileId());
bucketIdToFileIdPrefixMap.put(k, prefix);
fileIdPrefixToBucketIndex.put(prefix, fileIdPfxList.size());
fileIdPfxList.add(prefix);
diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala
index bfeb5df2848b..95156d417909 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala
@@ -144,7 +144,8 @@ class SparkFileFormatInternalRowReaderContext(baseFileReader: SparkColumnarFileR
// Parquet stores VECTOR as FIXED_LEN_BYTE_ARRAY, so the reader needs BinaryType
// and we decode back to ArrayType below. Lance returns ArrayType natively, so skip
// the rewrite only for Lance base files; log files always go through the rewrite path.
- val isLanceBaseFile = FSUtils.isBaseFile(filePath) &&
+ val isLogFile = FSUtils.isLogFile(filePath)
+ val isLanceBaseFile = !isLogFile && FSUtils.isBaseFile(filePath) &&
tableConfig.getBaseFileFormat == HoodieFileFormat.LANCE
val vectorColumnInfo: Map[Int, HoodieSchema.Vector] = if (isLanceBaseFile) {
Map.empty
@@ -158,7 +159,7 @@ class SparkFileFormatInternalRowReaderContext(baseFileReader: SparkColumnarFileR
}
val (readSchema, readFilters) = getSchemaAndFiltersForRead(parquetReadStructType, hasRowIndexField)
- if (FSUtils.isLogFile(filePath)) {
+ if (isLogFile) {
// NOTE: now only primary key based filtering is supported for log files
// Position-based merging pairs log records with the RECORD_POSITIONS bitmap by index (see
// PositionBasedFileGroupRecordBuffer), which requires the record stream to contain every
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkConsistentBucketClusteringPlanStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkConsistentBucketClusteringPlanStrategy.java
index 5c9a78b15a91..4e45545f1f7e 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkConsistentBucketClusteringPlanStrategy.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkConsistentBucketClusteringPlanStrategy.java
@@ -188,7 +188,7 @@ public class TestSparkConsistentBucketClusteringPlanStrategy extends HoodieSpark
String fileId = FSUtils.createNewFileId(fileIdPfx, 0);
FileSlice fs = new FileSlice("partition", "001", fileId);
if (baseFileSize > 0) {
- HoodieBaseFile f = new HoodieBaseFile(fileId);
+ HoodieBaseFile f = new HoodieBaseFile(FSUtils.makeBaseFileName("001", "1-0-1", fileId, ".parquet"));
f.setFileSize(baseFileSize);
fs.setBaseFile(f);
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java
index ebcc81417de8..b9d69f0d3a48 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java
@@ -309,7 +309,7 @@ public class TestSparkSizeBasedClusteringPlanStrategy {
private FileSlice createFileSliceWithCommitTime(long baseFileSize, String commitTime) {
String fileId = FSUtils.createNewFileId(FSUtils.createNewFileIdPfx(), 0);
FileSlice fs = new FileSlice("p0", commitTime, fileId);
- String basePath = "/test/path/" + fileId + "_" + commitTime + ".parquet";
+ String basePath = "/test/path/" + FSUtils.makeBaseFileName(commitTime, "1-0-1", fileId, ".parquet");
HoodieBaseFile f = new HoodieBaseFile(basePath);
f.setFileSize(baseFileSize);
fs.setBaseFile(f);
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
index 3102e3c42f99..86846d51db79 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
@@ -25,9 +25,9 @@ import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieLogFile;
+import org.apache.hudi.common.model.LogExtensions;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.hudi.common.table.cdc.HoodieCDCUtils;
import org.apache.hudi.common.util.HoodieStorageUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
@@ -35,7 +35,6 @@ import org.apache.hudi.common.util.collection.ImmutablePair;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
-import org.apache.hudi.exception.HoodieValidationException;
import org.apache.hudi.exception.InvalidHoodiePathException;
import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.storage.HoodieStorage;
@@ -63,8 +62,6 @@ import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import java.util.function.Predicate;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -73,21 +70,7 @@ import java.util.stream.Stream;
*/
@Slf4j
public class FSUtils {
-
- // Log files are of this pattern - .b5068208-e1a4-11e6-bf01-fe55135034f3_20170101134598.log.1_1-0-1
- // Archive log files are of this pattern - .commits_.archive.1_1-0-1
- // Native log files are of this pattern - b5068208-e1a4-11e6-bf01-fe55135034f3_1-0-1_20170101134598_1.log.parquet
- // For native log files, the file extension is log/deletes/cdc and the suffix is the native file format.
public static final String PATH_SEPARATOR = "/";
- public static final Pattern LOG_FILE_PATTERN =
- Pattern.compile("^\\.([^._]+)_([^.]*)\\.(log|archive)\\.(\\d+)(_((\\d+)-(\\d+)-(\\d+))(\\.cdc)?)?$");
- public static final Pattern NATIVE_LOG_FILE_PATTERN =
- Pattern.compile("^([^._]+)_((\\d+)-(\\d+)-(\\d+))_([^_]+)_(\\d+)\\.(log|deletes|cdc)\\.([^.]+)$");
- public static final Pattern PREFIX_BY_FILE_ID_PATTERN = Pattern.compile("^(.+)-(\\d+)");
- private static final Pattern BASE_FILE_PATTERN = Pattern.compile("[a-zA-Z0-9-]+_[a-zA-Z0-9-]+_[0-9]+\\.[a-zA-Z0-9]+");
-
- private static final String LOG_FILE_EXTENSION = "log";
- private static final String LOG_FILE_START_WITH_CHARACTER = ".";
private static final StoragePathFilter ALLOW_ALL_FILTER = file -> true;
@@ -134,18 +117,17 @@ public class FSUtils {
}
public static String getCommitTime(String fullFileName) {
- try {
- Option<Matcher> nativeLogMatcher = matchNativeLogFile(fullFileName);
- if (nativeLogMatcher.isPresent()) {
- return nativeLogMatcher.get().group(6);
- }
- if (isLogFile(fullFileName)) {
- return fullFileName.split("_")[1].split("\\.", 2)[0];
- }
- return fullFileName.split("_")[2].split("\\.", 2)[0];
- } catch (ArrayIndexOutOfBoundsException e) {
- throw new HoodieException("Failed to get commit time from filename: " + fullFileName, e);
+ Option<FileNameParser.LogFileName> logFileName = FileNameParser.parseLogFile(fullFileName);
+ if (logFileName.isPresent()) {
+ return logFileName.get().getDeltaCommitTime();
}
+
+ Option<FileNameParser.BaseFileName> baseFileName = FileNameParser.parseBaseFile(fullFileName);
+ if (baseFileName.isPresent()) {
+ return baseFileName.get().getCommitTime();
+ }
+
+ throw new HoodieException("Failed to get commit time from filename: " + fullFileName);
}
public static String getCommitTimeWithFullPath(String path) {
@@ -301,17 +283,6 @@ public class FSUtils {
return UUID.randomUUID().toString();
}
- /**
- * Returns prefix for a file group from fileId.
- */
- public static String getFileIdPfxFromFileId(String fileId) {
- Matcher matcher = PREFIX_BY_FILE_ID_PATTERN.matcher(fileId);
- if (!matcher.find()) {
- throw new HoodieValidationException("Failed to get prefix from " + fileId);
- }
- return matcher.group(1);
- }
-
public static String createNewFileId(String idPfx, int id) {
// format: {idPrefix}-{id}
return new StringBuilder()
@@ -336,35 +307,27 @@ public class FSUtils {
* Get the file extension from the log file.
*/
public static String getFileExtensionFromLog(StoragePath logPath) {
- Option<Matcher> nativeLogMatcher = matchNativeLogFile(logPath.getName());
- if (nativeLogMatcher.isPresent()) {
- return nativeLogMatcher.get().group(8);
- }
- Matcher matcher = LOG_FILE_PATTERN.matcher(logPath.getName());
- if (!matcher.matches()) {
+ Option<FileNameParser.LogFileName> logFileName = FileNameParser.parseLogFile(logPath.getName());
+ if (!logFileName.isPresent()) {
throw new InvalidHoodiePathException(logPath.toString(), "LogFile");
}
- return matcher.group(3);
+ return logFileName.get().getFileExtension();
}
public static String getFileIdFromFileName(String fileName) {
- Option<Matcher> logFileMatcher = matchLogFile(fileName);
- if (logFileMatcher.isPresent()) {
- return logFileMatcher.get().group(1);
+ Option<FileNameParser.LogFileName> logFileName = FileNameParser.parseLogFile(fileName);
+ if (logFileName.isPresent()) {
+ return logFileName.get().getFileId();
}
return FSUtils.getFileId(fileName);
}
public static String getFileIdFromLogPath(StoragePath path) {
- Option<Matcher> nativeLogMatcher = matchNativeLogFile(path.getName());
- if (nativeLogMatcher.isPresent()) {
- return nativeLogMatcher.get().group(1);
- }
- Option<Matcher> logFileMatcher = matchLogFile(path.getName());
- if (!logFileMatcher.isPresent()) {
+ Option<FileNameParser.LogFileName> logFileName = FileNameParser.parseLogFile(path.getName());
+ if (!logFileName.isPresent()) {
throw new InvalidHoodiePathException(path, "LogFile");
}
- return logFileMatcher.get().group(1);
+ return logFileName.get().getFileId();
}
public static String getFileIdFromFilePath(StoragePath filePath) {
@@ -378,78 +341,55 @@ public class FSUtils {
* Get the second part of the file name in the log file. That will be the delta commit time.
*/
public static String getDeltaCommitTimeFromLogPath(StoragePath path) {
- Option<Matcher> nativeLogMatcher = matchNativeLogFile(path.getName());
- if (nativeLogMatcher.isPresent()) {
- return nativeLogMatcher.get().group(6);
- }
- Matcher matcher = LOG_FILE_PATTERN.matcher(path.getName());
- if (!matcher.matches()) {
+ Option<FileNameParser.LogFileName> logFileName = FileNameParser.parseLogFile(path.getName());
+ if (!logFileName.isPresent()) {
throw new InvalidHoodiePathException(path.toString(), "LogFile");
}
- return matcher.group(2);
+ return logFileName.get().getDeltaCommitTime();
}
/**
* Get TaskPartitionId used in log-path.
*/
public static Integer getTaskPartitionIdFromLogPath(StoragePath path) {
- Option<Matcher> nativeLogMatcher = matchNativeLogFile(path.getName());
- if (nativeLogMatcher.isPresent()) {
- return Integer.parseInt(nativeLogMatcher.get().group(3));
- }
- Matcher matcher = LOG_FILE_PATTERN.matcher(path.getName());
- if (!matcher.matches()) {
+ Option<FileNameParser.LogFileName> logFileName = FileNameParser.parseLogFile(path.getName());
+ if (!logFileName.isPresent()) {
throw new InvalidHoodiePathException(path.toString(), "LogFile");
}
- String val = matcher.group(7);
- return val == null ? null : Integer.parseInt(val);
+ return logFileName.get().getTaskPartitionId();
}
/**
* Get Write-Token used in log-path.
*/
public static String getWriteTokenFromLogPath(StoragePath path) {
- Option<Matcher> nativeLogMatcher = matchNativeLogFile(path.getName());
- if (nativeLogMatcher.isPresent()) {
- return nativeLogMatcher.get().group(2);
- }
- Matcher matcher = LOG_FILE_PATTERN.matcher(path.getName());
- if (!matcher.matches()) {
+ Option<FileNameParser.LogFileName> logFileName = FileNameParser.parseLogFile(path.getName());
+ if (!logFileName.isPresent()) {
throw new InvalidHoodiePathException(path.toString(), "LogFile");
}
- return matcher.group(6);
+ return logFileName.get().getWriteToken();
}
/**
* Get StageId used in log-path.
*/
public static Integer getStageIdFromLogPath(StoragePath path) {
- Option<Matcher> nativeLogMatcher = matchNativeLogFile(path.getName());
- if (nativeLogMatcher.isPresent()) {
- return Integer.parseInt(nativeLogMatcher.get().group(4));
- }
- Matcher matcher = LOG_FILE_PATTERN.matcher(path.getName());
- if (!matcher.matches()) {
+ Option<FileNameParser.LogFileName> logFileName = FileNameParser.parseLogFile(path.getName());
+ if (!logFileName.isPresent()) {
throw new InvalidHoodiePathException(path.toString(), "LogFile");
}
- String val = matcher.group(8);
- return val == null ? null : Integer.parseInt(val);
+ return logFileName.get().getStageId();
}
/**
* Get Task Attempt Id used in log-path.
*/
public static Integer getTaskAttemptIdFromLogPath(StoragePath path) {
- Option<Matcher> nativeLogMatcher = matchNativeLogFile(path.getName());
- if (nativeLogMatcher.isPresent()) {
- return Integer.parseInt(nativeLogMatcher.get().group(5));
- }
- Matcher matcher = LOG_FILE_PATTERN.matcher(path.getName());
- if (!matcher.matches()) {
+ Option<FileNameParser.LogFileName> logFileName = FileNameParser.parseLogFile(path.getName());
+ if (!logFileName.isPresent()) {
throw new InvalidHoodiePathException(path.toString(), "LogFile");
}
- String val = matcher.group(9);
- return val == null ? null : Integer.parseInt(val);
+ return logFileName.get().getTaskAttemptId();
}
/**
@@ -460,19 +400,15 @@ public class FSUtils {
}
public static int getFileVersionFromLog(String logFileName) {
- Option<Matcher> nativeLogMatcher = matchNativeLogFile(logFileName);
- if (nativeLogMatcher.isPresent()) {
- return Integer.parseInt(nativeLogMatcher.get().group(7));
- }
- Matcher matcher = LOG_FILE_PATTERN.matcher(logFileName);
- if (!matcher.matches()) {
+ Option<FileNameParser.LogFileName> parsedLogFileName = FileNameParser.parseLogFile(logFileName);
+ if (!parsedLogFileName.isPresent()) {
throw new HoodieIOException("Invalid log file name: " + logFileName);
}
- return Integer.parseInt(matcher.group(4));
+ return parsedLogFileName.get().getLogVersion();
}
- public static String makeLogFileName(String fileId, String logFileExtension, String deltaCommitTime, int version,
- String writeToken) {
+ public static String makeInlineLogFileName(String fileId, String logFileExtension, String deltaCommitTime, int version,
+ String writeToken) {
String suffix = (writeToken == null)
? String.format("%s_%s%s.%d", fileId, deltaCommitTime, logFileExtension, version)
: String.format("%s_%s%s.%d_%s", fileId, deltaCommitTime, logFileExtension, version, writeToken);
@@ -489,14 +425,10 @@ public class FSUtils {
}
public static boolean isBaseFile(String path) {
- if (matchNativeLogFile(path).isPresent()) {
- return false;
- }
- String extension = getFileExtension(path);
- if (HoodieFileFormat.BASE_FILE_EXTENSIONS.contains(extension)) {
- return BASE_FILE_PATTERN.matcher(path).matches();
- }
- return false;
+ return FileNameParser.parseBaseFile(path)
+ .map(baseFileName -> !isNativeLogFile(path)
+ && HoodieFileFormat.BASE_FILE_EXTENSIONS.contains(baseFileName.getFileExtension()))
+ .orElse(false);
}
public static boolean isBaseFile(StoragePath path) {
@@ -504,12 +436,11 @@ public class FSUtils {
}
public static String getWriteTokenFromBaseFile(String fileName) {
- Matcher matcher = BASE_FILE_PATTERN.matcher(fileName);
- if (!matcher.find()) {
+ Option<FileNameParser.BaseFileName> baseFileName = FileNameParser.parseBaseFile(fileName);
+ if (!baseFileName.isPresent()) {
throw new InvalidHoodiePathException(fileName, "BaseFile");
}
- String[] pathParts = fileName.split("_");
- return pathParts[1];
+ return baseFileName.get().getWriteToken();
}
public static boolean isLogFile(StoragePath logPath) {
@@ -519,58 +450,37 @@ public class FSUtils {
}
public static boolean isLogFile(String fileName) {
- if (matchNativeLogFile(fileName).isPresent()) {
- return true;
- }
- if (fileName.startsWith(LOG_FILE_START_WITH_CHARACTER)) {
- Matcher matcher = LOG_FILE_PATTERN.matcher(fileName);
- return matcher.matches() && matcher.group(3).equals(LOG_FILE_EXTENSION);
- }
- return false;
+ return FileNameParser.parseLogFile(fileName)
+ .map(logFileName -> logFileName.isNativeLogFile()
+ || logFileName.getFileExtension().equals(LogExtensions.DATA_LOG_EXTENSION))
+ .orElse(false);
}
public static boolean isInlineLogFile(String fileName) {
- return matchNativeLogFile(fileName).isEmpty();
+ return !isNativeLogFile(fileName);
}
- public static Option<Matcher> matchNativeLogFile(String fileName) {
- if (StringUtils.isNullOrEmpty(fileName)) {
- return Option.empty();
- }
- String actualFileName = fileName.contains(StoragePath.SEPARATOR)
- ? fileName.substring(fileName.lastIndexOf(StoragePath.SEPARATOR) + 1)
- : fileName;
- Matcher matcher = NATIVE_LOG_FILE_PATTERN.matcher(actualFileName);
- return matcher.matches() ? Option.of(matcher) : Option.empty();
+ public static boolean isNativeLogFile(String fileName) {
+ return FileNameParser.parseNativeLogFile(fileName).isPresent();
}
public static boolean isNativeDeleteLogFile(String fileName) {
- return matchNativeLogFile(fileName).map(matcher -> "deletes".equals(matcher.group(8))).orElse(false);
+ return FileNameParser.parseNativeLogFile(fileName)
+ .map(logFileName -> LogExtensions.DELETE_LOG_EXTENSION.equals(logFileName.getFileExtension()))
+ .orElse(false);
}
public static boolean isNativeCDCLogFile(String fileName) {
- return matchNativeLogFile(fileName).map(matcher -> "cdc".equals(matcher.group(8))).orElse(false);
+ return FileNameParser.parseNativeLogFile(fileName)
+ .map(logFileName -> LogExtensions.CDC_LOG_EXTENSION.equals(logFileName.getFileExtension()))
+ .orElse(false);
}
public static boolean isCDCLogFile(String fileName) {
if (StringUtils.isNullOrEmpty(fileName)) {
return false;
}
- Option<Matcher> nativeLogMatcher = matchNativeLogFile(fileName);
- if (nativeLogMatcher.isPresent()) {
- return HoodieCDCUtils.CDC_LOGFILE_SUFFIX.substring(1).equals(nativeLogMatcher.get().group(8));
- }
- Matcher matcher = LOG_FILE_PATTERN.matcher(getFileNameFromPath(fileName));
- return matcher.matches() && HoodieCDCUtils.CDC_LOGFILE_SUFFIX.equals(matcher.group(10));
- }
-
- private static Option<Matcher> matchLogFile(String fileName) {
- Option<Matcher> nativeLogMatcher = matchNativeLogFile(fileName);
- if (nativeLogMatcher.isPresent()) {
- return nativeLogMatcher;
- }
- Matcher matcher = LOG_FILE_PATTERN.matcher(fileName);
- return matcher.matches() ? Option.of(matcher) : Option.empty();
+ return FileNameParser.parseLogFile(fileName).map(FileNameParser.LogFileName::isCDCLogFile).orElse(false);
}
public static boolean isDataFile(StoragePath path) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FileNameParser.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FileNameParser.java
new file mode 100644
index 000000000000..16ef14bfcdd3
--- /dev/null
+++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FileNameParser.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.common.fs;
+
+import org.apache.hudi.common.table.cdc.HoodieCDCUtils;
+import org.apache.hudi.common.util.ExternalFilePathUtil;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.exception.HoodieValidationException;
+import org.apache.hudi.storage.StoragePath;
+
+import lombok.Getter;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Parser for Hudi base and log file names.
+ *
+ * <p>The parser accepts either a file name or a full path and always decodes only the last path component.
+ * A non-matching input returns {@link Option#empty()} instead of throwing.</p>
+ */
+public final class FileNameParser {
+ // Base files are of this pattern - <fg-id>_<write-token>_<instant>.<suffix>
+ // Inline log files are of this pattern - .<fg-id>_<instant>.log.<version>_<write-token>
+ // Inline archive log files are of this pattern - .commits_.archive.<version>_<write-token>
+ // Native log files are of this pattern - <fg-id>_<write-token>_<instant>_<version>.log.<suffix>
+ // For native log files, the file extension is log/deletes/cdc and the suffix is the native file format.
+
+ static final Pattern INLINE_LOG_FILE_PATTERN =
+ Pattern.compile("^\\.([^._]+)_([^.]*)\\.(log|archive)\\.(\\d+)(_((\\d+)-(\\d+)-(\\d+))(\\.cdc)?)?$");
+ static final Pattern NATIVE_LOG_FILE_PATTERN =
+ Pattern.compile("^([^._]+)_((\\d+)-(\\d+)-(\\d+))_([^_]+)_(\\d+)\\.(log|deletes|cdc)\\.([^.]+)$");
+ private static final Pattern PREFIX_BY_FILE_ID_PATTERN = Pattern.compile("^(.+)-(\\d+)");
+ private static final char UNDERSCORE = '_';
+ private static final char DOT = '.';
+
+ private FileNameParser() {
+ }
+
+ /**
+ * Parses a base file name.
+ *
+ * <p>A Hudi-generated base file name is encoded as
+ * {@code <fileId>_<writeToken>_<commitTime><fileExtension>}. For example,
+ * {@code 136281f3-c24e-423b-a65a-95dbfbddce1d_1-0-1_20240706120100123.parquet}
+ * has file id {@code 136281f3-c24e-423b-a65a-95dbfbddce1d}, write token {@code 1-0-1},
+ * commit time {@code 20240706120100123}, and file extension {@code .parquet}. The decoded
+ * {@link BaseFileName#getFileExtension()} value includes the leading dot when an extension is present.</p>
+ *
+ * <p>Decoding mirrors {@code HoodieBaseFile}'s historical substring scan for Hudi-generated names: the
+ * first underscore terminates the file id, the last underscore seen before a commit-time delimiter
+ * precedes the commit time, and the first dot after the second underscore terminates the commit time.
+ * If the second underscore is absent, the parser treats the name as a legacy base file without a write
+ * token and uses the first dot after the first underscore as the commit-time delimiter. If the delimiter
+ * dot is absent, the commit time extends to the end of the file name. This looser fallback is kept to
+ * preserve the historical {@code FSUtils} behavior for non-generated/external base file names such as
+ * {@code file_1.parquet}; it is not the naming pattern produced by Hudi writers.</p>
+ *
+ * <p>Externally registered base files marked with {@code _hudiext} are decoded through
+ * {@link ExternalFilePathUtil}. For these files, {@link BaseFileName#getWriteToken()} is empty and
+ * {@link BaseFileName#getFileExtension()} is derived from the original external file name.</p>
+ *
+ * @param fileName file name or full path
+ * @return decoded base file name when the input matches a supported base-file format
+ */
+ public static Option<BaseFileName> parseBaseFile(String fileName) {
+ if (StringUtils.isNullOrEmpty(fileName)) {
+ return Option.empty();
+ }
+
+ String actualFileName = getActualFileName(fileName);
+ if (ExternalFilePathUtil.isExternallyCreatedFile(actualFileName)) {
+ return parseExternalBaseFile(actualFileName);
+ }
+
+ int firstUnderscoreIndex = -1;
+ int secondUnderscoreIndex = -1;
+ int dotIndex = -1;
+ short underscoreCount = 0;
+ for (int i = 0; i < actualFileName.length(); i++) {
+ char c = actualFileName.charAt(i);
+ if (c == UNDERSCORE) {
+ if (underscoreCount == 0) {
+ firstUnderscoreIndex = i;
+ } else if (underscoreCount == 1) {
+ secondUnderscoreIndex = i;
+ }
+ underscoreCount++;
+ } else if (c == DOT && underscoreCount == 2) {
+ dotIndex = i;
+ break;
+ }
+ }
+ if (firstUnderscoreIndex < 0) {
+ return Option.empty();
+ }
+ if (secondUnderscoreIndex < 0) {
+ // Preserve the old loose base-file parsing used by FSUtils for external/non-generated file names.
+ int commitTimeEndIndex = actualFileName.indexOf(DOT, firstUnderscoreIndex + 1);
+ return Option.of(new BaseFileName(
+ actualFileName.substring(0, firstUnderscoreIndex),
+ "",
+ actualFileName.substring(firstUnderscoreIndex + 1, commitTimeEndIndex < 0 ? actualFileName.length() : commitTimeEndIndex),
+ commitTimeEndIndex < 0 ? "" : actualFileName.substring(commitTimeEndIndex)));
+ }
+ int commitTimeEndIndex = dotIndex < 0 ? actualFileName.length() : dotIndex;
+ return Option.of(new BaseFileName(
+ actualFileName.substring(0, firstUnderscoreIndex),
+ actualFileName.substring(firstUnderscoreIndex + 1, secondUnderscoreIndex),
+ actualFileName.substring(secondUnderscoreIndex + 1, commitTimeEndIndex),
+ dotIndex < 0 ? "" : actualFileName.substring(dotIndex)));
+ }
+
+ /**
+ * Parses either an inline log file name or a native log file name.
+ *
+ * <p>Inline log format: {@code .<fileId>_<deltaCommitTime>.<extension>.<version>_<writeToken>[.cdc]}.
+ * Native log format: {@code <fileId>_<writeToken>_<deltaCommitTime>_<version>.<extension>.<nativeFormat>}.
+ * The decoded {@link LogFileName#getFileExtension()} value does not include a leading dot. For inline CDC
+ * logs, {@link LogFileName#getSuffix()} is {@code .cdc}; for native logs, it is the native storage format,
+ * such as {@code parquet}.</p>
+ *
+ * @param fileName file name or full path
+ * @return decoded log file name when the input matches either supported log-file format
+ */
+ public static Option<LogFileName> parseLogFile(String fileName) {
+ if (StringUtils.isNullOrEmpty(fileName)) {
+ return Option.empty();
+ }
+
+ String actualFileName = getActualFileName(fileName);
+ Option<LogFileName> nativeLogFileName = parseNativeLogFileFromActualFileName(actualFileName);
+ if (nativeLogFileName.isPresent()) {
+ return nativeLogFileName;
+ }
+
+ Matcher matcher = INLINE_LOG_FILE_PATTERN.matcher(actualFileName);
+ return matcher.matches() ? Option.of(LogFileName.fromInlineLogFile(matcher)) : Option.empty();
+ }
+
+ /**
+ * Parses only a native log file name.
+ *
+ * <p>Use {@link #parseLogFile(String)} when both inline and native log formats are acceptable.</p>
+ *
+ * @param fileName file name or full path
+ * @return decoded native log file name when the input matches the native log-file format
+ */
+ public static Option<LogFileName> parseNativeLogFile(String fileName) {
+ if (StringUtils.isNullOrEmpty(fileName)) {
+ return Option.empty();
+ }
+
+ String actualFileName = getActualFileName(fileName);
+ return parseNativeLogFileFromActualFileName(actualFileName);
+ }
+
+ /**
+ * Returns the file group prefix encoded in a Hudi file id.
+ *
+ * @param fileId Hudi file id ending with a numeric suffix separated by {@code -}
+ * @return file id prefix before the trailing numeric suffix
+ * @throws HoodieValidationException when the file id does not contain the expected suffix
+ */
+ public static String getFileIdPfxFromFileId(String fileId) {
+ Matcher matcher = PREFIX_BY_FILE_ID_PATTERN.matcher(fileId);
+ if (!matcher.find()) {
+ throw new HoodieValidationException("Failed to get prefix from " + fileId);
+ }
+ return matcher.group(1);
+ }
+
+ private static Option<LogFileName> parseNativeLogFileFromActualFileName(String actualFileName) {
+ return matchNativeLogFileFromActualFileName(actualFileName).map(LogFileName::fromNativeLogFile);
+ }
+
+ private static Option<Matcher> matchNativeLogFileFromActualFileName(String actualFileName) {
+ Matcher matcher = NATIVE_LOG_FILE_PATTERN.matcher(actualFileName);
+ return matcher.matches() ? Option.of(matcher) : Option.empty();
+ }
+
+ private static String getActualFileName(String fileName) {
+ return fileName.contains(StoragePath.SEPARATOR)
+ ? fileName.substring(fileName.lastIndexOf(StoragePath.SEPARATOR) + 1)
+ : fileName;
+ }
+
+ private static Option<BaseFileName> parseExternalBaseFile(String actualFileName) {
+ String[] fileIdAndCommitTime = ExternalFilePathUtil.parseFileIdAndCommitTimeFromExternalFile(actualFileName);
+ return Option.of(new BaseFileName(fileIdAndCommitTime[0], "", fileIdAndCommitTime[1],
+ getFileExtension(fileIdAndCommitTime[0])));
+ }
+
+ private static String getFileExtension(String fileName) {
+ int lastSeparatorIndex = fileName.lastIndexOf(StoragePath.SEPARATOR);
+ int dotIndex = fileName.lastIndexOf(DOT);
+ return dotIndex > lastSeparatorIndex ? fileName.substring(dotIndex) : "";
+ }
+
+ /**
+ * Decoded parts of a base file name.
+ */
+ @Getter
+ public static final class BaseFileName {
+ private final String fileId;
+ private final String writeToken;
+ private final String commitTime;
+ private final String fileExtension;
+
+ private BaseFileName(String fileId, String writeToken, String commitTime, String fileExtension) {
+ this.fileId = fileId;
+ this.writeToken = writeToken;
+ this.commitTime = commitTime;
+ this.fileExtension = fileExtension;
+ }
+
+ }
+
+ /**
+ * Decoded parts of a Hudi-generated log file name.
+ */
+ @Getter
+ public static final class LogFileName {
+ private final String fileId;
+ private final String deltaCommitTime;
+ private final int logVersion;
+ private final String writeToken;
+ private final Integer taskPartitionId;
+ private final Integer stageId;
+ private final Integer taskAttemptId;
+ private final String fileExtension;
+ // Inline logs use this as the optional CDC suffix (".cdc"), while native logs use it as
+ // the native storage format suffix, such as "parquet".
+ private final String suffix;
+ private final boolean nativeLogFile;
+
+ private LogFileName(String fileId, String deltaCommitTime, int logVersion, String writeToken,
+ Integer taskPartitionId, Integer stageId, Integer taskAttemptId,
+ String fileExtension, String suffix, boolean nativeLogFile) {
+ this.fileId = fileId;
+ this.deltaCommitTime = deltaCommitTime;
+ this.logVersion = logVersion;
+ this.writeToken = writeToken;
+ this.taskPartitionId = taskPartitionId;
+ this.stageId = stageId;
+ this.taskAttemptId = taskAttemptId;
+ this.fileExtension = fileExtension;
+ this.suffix = suffix;
+ this.nativeLogFile = nativeLogFile;
+ }
+
+ private static LogFileName fromInlineLogFile(Matcher matcher) {
+ String taskPartitionId = matcher.group(7);
+ String stageId = matcher.group(8);
+ String taskAttemptId = matcher.group(9);
+ return new LogFileName(
+ matcher.group(1),
+ matcher.group(2),
+ Integer.parseInt(matcher.group(4)),
+ matcher.group(6),
+ taskPartitionId == null ? null : Integer.parseInt(taskPartitionId),
+ stageId == null ? null : Integer.parseInt(stageId),
+ taskAttemptId == null ? null : Integer.parseInt(taskAttemptId),
+ matcher.group(3),
+ matcher.group(10) == null ? "" : matcher.group(10),
+ false);
+ }
+
+ private static LogFileName fromNativeLogFile(Matcher matcher) {
+ return new LogFileName(
+ matcher.group(1),
+ matcher.group(6),
+ Integer.parseInt(matcher.group(7)),
+ matcher.group(2),
+ Integer.parseInt(matcher.group(3)),
+ Integer.parseInt(matcher.group(4)),
+ Integer.parseInt(matcher.group(5)),
+ matcher.group(8),
+ matcher.group(9),
+ true);
+ }
+
+ public boolean isCDCLogFile() {
+ return nativeLogFile
+ ? HoodieCDCUtils.CDC_LOGFILE_SUFFIX.substring(1).equals(fileExtension)
+ : HoodieCDCUtils.CDC_LOGFILE_SUFFIX.equals(suffix);
+ }
+ }
+}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieBaseFile.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieBaseFile.java
index 96095f4a5be0..2ee0385ca5e6 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieBaseFile.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieBaseFile.java
@@ -18,8 +18,10 @@
package org.apache.hudi.common.model;
+import org.apache.hudi.common.fs.FileNameParser;
import org.apache.hudi.common.util.ExternalFilePathUtil;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.exception.InvalidHoodiePathException;
import org.apache.hudi.storage.StoragePathInfo;
import lombok.Getter;
@@ -34,8 +36,6 @@ import lombok.ToString;
public class HoodieBaseFile extends BaseFile {
private static final long serialVersionUID = 1L;
- private static final char UNDERSCORE = '_';
- private static final char DOT = '.';
private final String fileId;
private final String commitTime;
@@ -95,35 +95,15 @@ public class HoodieBaseFile extends BaseFile {
* @return String array of size 2 with fileId as the first and commitTime as the second element.
*/
private static String[] getFileIdAndCommitTimeFromFileName(String fileName) {
- return ExternalFilePathUtil.isExternallyCreatedFile(fileName) ? handleExternallyGeneratedFile(fileName) : handleHudiGeneratedFile(fileName);
- }
-
- private static String[] handleHudiGeneratedFile(String fileName) {
- String[] values = new String[2];
- short underscoreCount = 0;
- short lastUnderscoreIndex = 0;
- for (int i = 0; i < fileName.length(); i++) {
- char c = fileName.charAt(i);
- if (c == UNDERSCORE) {
- if (underscoreCount == 0) {
- values[0] = fileName.substring(0, i);
- }
- lastUnderscoreIndex = (short) i;
- underscoreCount++;
- } else if (c == DOT) {
- if (underscoreCount == 2) {
- values[1] = fileName.substring(lastUnderscoreIndex + 1, i);
- return values;
- }
- }
+ Option<FileNameParser.BaseFileName> parsedBaseFileName = FileNameParser.parseBaseFile(fileName);
+ if (!parsedBaseFileName.isPresent()) {
+ throw new InvalidHoodiePathException(fileName, "BaseFile");
}
- // case where there is no '.' in file name (no file suffix like .parquet)
- values[1] = fileName.substring(lastUnderscoreIndex + 1);
- return values;
- }
- private static String[] handleExternallyGeneratedFile(String fileName) {
- return ExternalFilePathUtil.parseFileIdAndCommitTimeFromExternalFile(fileName);
+ return new String[] {
+ parsedBaseFileName.get().getFileId(),
+ parsedBaseFileName.get().getCommitTime()
+ };
}
public void setBootstrapBaseFile(BaseFile bootstrapBaseFile) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java
index 9f5815937777..08bbe19866ef 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java
@@ -19,6 +19,7 @@
package org.apache.hudi.common.model;
import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.fs.FileNameParser;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.InvalidHoodiePathException;
import org.apache.hudi.storage.StoragePath;
@@ -34,9 +35,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
-import java.util.regex.Matcher;
-
-import static org.apache.hudi.common.fs.FSUtils.LOG_FILE_PATTERN;
/**
* Abstracts a single log file. Contains methods to extract metadata like
@@ -79,6 +77,8 @@ public class HoodieLogFile implements Serializable {
private String logWriteToken;
private String fileExtension;
private String suffix;
+ private Boolean nativeLogFile;
+ private Boolean cdcLogFile;
@Getter
@Setter
@ToString.Include
@@ -94,6 +94,8 @@ public class HoodieLogFile implements Serializable {
this.logWriteToken = logFile.getLogWriteToken();
this.fileExtension = logFile.getFileExtension();
this.suffix = logFile.getSuffix();
+ this.nativeLogFile = logFile.isNativeLogFile();
+ this.cdcLogFile = logFile.isCDC();
this.fileSize = logFile.getFileSize();
}
@@ -122,27 +124,19 @@ public class HoodieLogFile implements Serializable {
}
private void parseFieldsFromPath() {
- Option<Matcher> nativeLogMatcherOpt = FSUtils.matchNativeLogFile(getPath().getName());
- if (nativeLogMatcherOpt.isPresent()) {
- Matcher matcher = nativeLogMatcherOpt.get();
- this.fileId = matcher.group(1);
- this.deltaCommitTime = matcher.group(6);
- this.fileExtension = matcher.group(8);
- this.logVersion = Integer.parseInt(matcher.group(7));
- this.logWriteToken = matcher.group(2);
- this.suffix = matcher.group(9);
- return;
- }
- Matcher matcher = LOG_FILE_PATTERN.matcher(getPath().getName());
- if (!matcher.matches()) {
+ Option<FileNameParser.LogFileName> parsedLogFileName = FileNameParser.parseLogFile(getPath().getName());
+ if (!parsedLogFileName.isPresent()) {
throw new InvalidHoodiePathException(path, "LogFile");
}
- this.fileId = matcher.group(1);
- this.deltaCommitTime = matcher.group(2);
- this.fileExtension = matcher.group(3);
- this.logVersion = Integer.parseInt(matcher.group(4));
- this.logWriteToken = matcher.group(6);
- this.suffix = matcher.group(10) == null ? "" : matcher.group(10);
+ FileNameParser.LogFileName logFileName = parsedLogFileName.get();
+ this.fileId = logFileName.getFileId();
+ this.deltaCommitTime = logFileName.getDeltaCommitTime();
+ this.fileExtension = logFileName.getFileExtension();
+ this.logVersion = logFileName.getLogVersion();
+ this.logWriteToken = logFileName.getWriteToken();
+ this.suffix = logFileName.getSuffix();
+ this.nativeLogFile = logFileName.isNativeLogFile();
+ this.cdcLogFile = logFileName.isCDCLogFile();
}
public String getFileId() {
@@ -181,7 +175,10 @@ public class HoodieLogFile implements Serializable {
}
public boolean isCDC() {
- return FSUtils.isCDCLogFile(getFileName());
+ if (cdcLogFile == null) {
+ parseFieldsFromPath();
+ }
+ return cdcLogFile;
}
public String getSuffix() {
@@ -191,6 +188,13 @@ public class HoodieLogFile implements Serializable {
return suffix;
}
+ public boolean isNativeLogFile() {
+ if (nativeLogFile == null) {
+ parseFieldsFromPath();
+ }
+ return nativeLogFile;
+ }
+
public StoragePath getPath() {
if (path == null) {
path = new StoragePath(pathStr);
@@ -206,14 +210,14 @@ public class HoodieLogFile implements Serializable {
String fileId = getFileId();
String deltaCommitTime = getDeltaCommitTime();
StoragePath path = getPath();
- if (FSUtils.matchNativeLogFile(path.getName()).isPresent()) {
+ if (isNativeLogFile()) {
return new HoodieLogFile(new StoragePath(path.getParent(),
FSUtils.makeNativeLogFileName(fileId, logWriteToken, deltaCommitTime, logVersion + 1,
fileExtension, HoodieFileFormat.fromFileExtension("." + getSuffix()))));
}
String extension = "." + fileExtension;
return new HoodieLogFile(new StoragePath(path.getParent(),
- FSUtils.makeLogFileName(fileId, extension, deltaCommitTime, logVersion + 1, logWriteToken)));
+ FSUtils.makeInlineLogFileName(fileId, extension, deltaCommitTime, logVersion + 1, logWriteToken)));
}
public static Comparator<HoodieLogFile> getLogFileComparator() {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
index ec3aa591c03c..1e8988a742cb 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
@@ -19,6 +19,7 @@
package org.apache.hudi.common.table;
import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.fs.FileNameParser;
import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieRecord;
@@ -61,7 +62,6 @@ import java.util.List;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Supplier;
-import java.util.regex.Matcher;
import java.util.stream.Stream;
/**
@@ -287,9 +287,9 @@ public class TableSchemaResolver {
* @return
*/
public static HoodieSchema readSchemaFromLogFile(HoodieTableMetaClient metaClient, StoragePath path) throws IOException {
- Option<Matcher> nativeLogMatcherOpt = FSUtils.matchNativeLogFile(path.getName());
- if (nativeLogMatcherOpt.isPresent()) {
- return NativeLogFooterMetadata.readSchemaFromNativeLogFile(metaClient.getStorage(), path, nativeLogMatcherOpt.get());
+ Option<FileNameParser.LogFileName> nativeLogFileName = FileNameParser.parseNativeLogFile(path.getName());
+ if (nativeLogFileName.isPresent()) {
+ return NativeLogFooterMetadata.readSchemaFromNativeLogFile(metaClient.getStorage(), path, nativeLogFileName.get());
}
// We only need to read the schema from the log block header,
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieLogRecordReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieLogRecordReader.java
index 170590df8829..591639e3aae0 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieLogRecordReader.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieLogRecordReader.java
@@ -21,7 +21,6 @@ package org.apache.hudi.common.table.log;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.engine.HoodieReaderContext;
-import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodiePayloadProps;
import org.apache.hudi.common.model.HoodieRecord;
@@ -41,7 +40,6 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.StoragePath;
import lombok.AccessLevel;
import lombok.Getter;
@@ -456,10 +454,10 @@ public abstract class BaseHoodieLogRecordReader<T> {
}
if (!logFiles.isEmpty()) {
try {
- StoragePath path = logFiles.get(0).getPath();
+ HoodieLogFile logFile = logFiles.get(0);
log.info("Finished scanning log files. FileId: {}, LogFileInstantTime: {}, "
+ "Total log files: {}, Total log blocks: {}, Total rollbacks: {}, Total corrupt blocks: {}",
- FSUtils.getFileIdFromLogPath(path), FSUtils.getDeltaCommitTimeFromLogPath(path),
+ logFile.getFileId(), logFile.getDeltaCommitTime(),
totalLogFiles.get(), totalLogBlocks.get(), totalRollbacks.get(), totalCorruptBlocks.get());
} catch (Exception e) {
log.warn("Could not extract fileId from log path", e);
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java
index e338e32a6283..33c4c005440d 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java
@@ -185,7 +185,7 @@ public interface HoodieLogFormat {
// Initialise logFile
StoragePath logPath = new StoragePath(parentPath,
- FSUtils.makeLogFileName(this.logFileId, this.fileExtension, this.instantTime, this.logVersion, this.logWriteToken));
+ FSUtils.makeInlineLogFileName(this.logFileId, this.fileExtension, this.instantTime, this.logVersion, this.logWriteToken));
log.info("HoodieLogFile on path {}", logPath);
this.logFile = new HoodieLogFile(logPath, this.fileSize);
}
@@ -250,7 +250,7 @@ public interface HoodieLogFormat {
static HoodieLogFormat.Reader newReader(HoodieTableMetaClient metaClient, HoodieLogFile logFile,
HoodieSchema readerSchema, boolean reverseReader) throws IOException {
HoodieStorage storage = metaClient.getStorage();
- if (FSUtils.matchNativeLogFile(logFile.getFileName()).isPresent()) {
+ if (FSUtils.isNativeLogFile(logFile.getFileName())) {
StoragePath logFileParent = logFile.getPath().getParent();
return new HoodieNativeLogFileReader(
storage,
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java
index e9fd5f2afc18..3c21e1cfcd0f 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java
@@ -125,7 +125,7 @@ public class HoodieLogFormatReader implements HoodieLogFormat.Reader {
}
private HoodieLogFormat.Reader createReader(HoodieLogFile logFile, boolean reverseLogReader) throws IOException {
- if (FSUtils.matchNativeLogFile(logFile.getFileName()).isPresent()) {
+ if (FSUtils.isNativeLogFile(logFile.getFileName())) {
if (metaClient == null) {
throw new HoodieNotSupportedException("Native log files require HoodieTableMetaClient");
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/NativeLogFooterMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/NativeLogFooterMetadata.java
index 0c9fc688af29..b5286c0bcb0a 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/NativeLogFooterMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/NativeLogFooterMetadata.java
@@ -19,7 +19,9 @@
package org.apache.hudi.common.table.log;
+import org.apache.hudi.common.fs.FileNameParser;
import org.apache.hudi.common.model.HoodieFileFormat;
+import org.apache.hudi.common.model.LogExtensions;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType;
import org.apache.hudi.common.util.JsonUtils;
@@ -35,7 +37,6 @@ import com.fasterxml.jackson.core.type.TypeReference;
import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.Map;
-import java.util.regex.Matcher;
/**
* Shared on-disk contract for RFC-103 native log files. The log block header (schema, instant time,
@@ -119,17 +120,16 @@ public class NativeLogFooterMetadata {
*
* @param storage storage used to read the native log file footer
* @param path native log file path
- * @param nativeLogMatcher matcher returned by {@code FSUtils.matchNativeLogFile(path.getName())}
+ * @param nativeLogFileName parsed native log file name
* @return table schema for native data logs, or {@code null} for native non-data logs
*/
public static HoodieSchema readSchemaFromNativeLogFile(
- HoodieStorage storage, StoragePath path, Matcher nativeLogMatcher) {
- String nativeLogType = nativeLogMatcher.group(8);
- if (!"log".equals(nativeLogType)) {
+ HoodieStorage storage, StoragePath path, FileNameParser.LogFileName nativeLogFileName) {
+ if (!LogExtensions.DATA_LOG_EXTENSION.equals(nativeLogFileName.getFileExtension())) {
return null;
}
- HoodieFileFormat fileFormat = HoodieFileFormat.fromFileExtension("." + nativeLogMatcher.group(9));
+ HoodieFileFormat fileFormat = HoodieFileFormat.fromFileExtension("." + nativeLogFileName.getSuffix());
Map<String, String> footer = HoodieIOFactory.getIOFactory(storage)
.getFileFormatUtils(fileFormat)
.readFooter(storage, false, path, FOOTER_METADATA_KEY);
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestFileSlice.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestFileSlice.java
index e8763d6b59f5..daf02bd0edd1 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestFileSlice.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestFileSlice.java
@@ -38,6 +38,8 @@ public class TestFileSlice {
private static final String PARTITION_PATH = "test_partition";
private static final String FILE_ID = "test_file_id";
private static final String BASE_INSTANT = "001";
+ private static final String BASE_FILE_NAME =
+ "b5068208-e1a4-11e6-bf01-fe55135034f3_1-0-1_" + BASE_INSTANT + ".parquet";
@Test
void testGetLatestInstantTime() {
@@ -63,7 +65,7 @@ public class TestFileSlice {
public void testGetAllFilesWithBaseFileOnly() {
// Create a FileSlice with only a base file and no log files
HoodieBaseFile baseFile = new HoodieBaseFile(
- "file://" + PARTITION_PATH + "/test_base_file.parquet");
+ "file://" + PARTITION_PATH + "/" + BASE_FILE_NAME);
FileSlice fileSlice = new FileSlice(
new HoodieFileGroupId(PARTITION_PATH, FILE_ID),
BASE_INSTANT,
@@ -100,7 +102,7 @@ public class TestFileSlice {
public void testGetAllFilesWithBaseFileAndLogFiles() {
// Create a FileSlice with both base file and log files
HoodieBaseFile baseFile = new HoodieBaseFile(
- "file://" + PARTITION_PATH + "/test_base_file.parquet");
+ "file://" + PARTITION_PATH + "/" + BASE_FILE_NAME);
// Log files must follow the proper naming convention: .{fileId}_{instant}.log.{version}
HoodieLogFile logFile1 = new HoodieLogFile(new StoragePath(PARTITION_PATH + "/." + FILE_ID + "_004.log.1"));
HoodieLogFile logFile2 = new HoodieLogFile(new StoragePath(PARTITION_PATH + "/." + FILE_ID + "_005.log.2"));
@@ -132,7 +134,7 @@ public class TestFileSlice {
public void testFilterLogFiles() {
// Create a FileSlice with both base file and log files
HoodieBaseFile baseFile = new HoodieBaseFile(
- "file://" + PARTITION_PATH + "/test_base_file.parquet");
+ "file://" + PARTITION_PATH + "/" + BASE_FILE_NAME);
// Log files must follow the proper naming convention: .{fileId}_{instant}.log.{version}
HoodieLogFile logFile1 = new HoodieLogFile(new StoragePath(PARTITION_PATH + "/." + FILE_ID + "_004.log.1"));
HoodieLogFile logFile2 = new HoodieLogFile(new StoragePath(PARTITION_PATH + "/." + FILE_ID + "_005.log.2"));
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieBaseFile.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieBaseFile.java
index d04cb5b6ce83..3ed22154efd2 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieBaseFile.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieBaseFile.java
@@ -18,6 +18,7 @@
package org.apache.hudi.common.model;
+import org.apache.hudi.common.util.ExternalFilePathUtil;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -70,6 +71,13 @@ public class TestHoodieBaseFile {
assertFileGetters(null, hoodieBaseFile, -1, Option.of(bootstrapBaseFile));
}
+ @Test
+ void createFromBaseFileWithoutExtension() {
+ HoodieBaseFile hoodieBaseFile = new HoodieBaseFile("136281f3-c24e-423b-a65a-95dbfbddce1d_1-0-1_100");
+ assertEquals("136281f3-c24e-423b-a65a-95dbfbddce1d", hoodieBaseFile.getFileId());
+ assertEquals("100", hoodieBaseFile.getCommitTime());
+ }
+
@Test
void createFromExternalFileStatus() {
String fileName = "parquet_file_1.parquet";
@@ -86,6 +94,40 @@ public class TestHoodieBaseFile {
expectedPathString, fileName);
}
+ @Test
+ void createFromExternalFileStatusWithFileGroupPrefix() {
+ String prefix = "bucket-0";
+ String fileName = "parquet_file_1.parquet";
+ String markedFileName = ExternalFilePathUtil.appendCommitTimeAndExternalFileMarker(fileName, baseCommitTime, prefix);
+ String storedPathString = "file:/tmp/hoodie/2021/01/01/" + prefix + "/" + markedFileName;
+ String expectedPathString = "file:/tmp/hoodie/2021/01/01/" + prefix + "/" + fileName;
+ StoragePathInfo inputPathInfo = new StoragePathInfo(
+ new StoragePath(storedPathString), length, false, blockReplication, blockSize, 0);
+ StoragePathInfo expectedPathInfo = new StoragePathInfo(
+ new StoragePath(expectedPathString), length, false, blockReplication, blockSize, 0);
+ HoodieBaseFile hoodieBaseFile = new HoodieBaseFile(inputPathInfo);
+
+ assertFileGetters(expectedPathInfo, hoodieBaseFile, length, Option.empty(), prefix + "/" + fileName,
+ expectedPathString, fileName);
+ }
+
+ @Test
+ void createFromExternalFileStatusWithFileGroupPrefixForUnpartitionedTable() {
+ String prefix = "bucket-0";
+ String fileName = "parquet_file_1.parquet";
+ String markedFileName = ExternalFilePathUtil.appendCommitTimeAndExternalFileMarker(fileName, baseCommitTime, prefix);
+ String storedPathString = "file:/tmp/hoodie/" + prefix + "/" + markedFileName;
+ String expectedPathString = "file:/tmp/hoodie/" + prefix + "/" + fileName;
+ StoragePathInfo inputPathInfo = new StoragePathInfo(
+ new StoragePath(storedPathString), length, false, blockReplication, blockSize, 0);
+ StoragePathInfo expectedPathInfo = new StoragePathInfo(
+ new StoragePath(expectedPathString), length, false, blockReplication, blockSize, 0);
+ HoodieBaseFile hoodieBaseFile = new HoodieBaseFile(inputPathInfo);
+
+ assertFileGetters(expectedPathInfo, hoodieBaseFile, length, Option.empty(), prefix + "/" + fileName,
+ expectedPathString, fileName);
+ }
+
private void assertFileGetters(StoragePathInfo pathInfo, HoodieBaseFile hoodieBaseFile,
long fileLength, Option<HoodieBaseFile> bootstrapBaseFile) {
assertFileGetters(pathInfo, hoodieBaseFile, fileLength, bootstrapBaseFile, fileId, pathStr,
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieLogFile.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieLogFile.java
index 1856d696749c..d041b6a850ec 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieLogFile.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieLogFile.java
@@ -146,6 +146,7 @@ public class TestHoodieLogFile {
assertEquals("parquet", hoodieLogFile.getSuffix());
assertEquals("cdc", FSUtils.getFileExtensionFromLog(nativeCdcLogPath));
assertTrue(hoodieLogFile.isCDC());
+ assertTrue(FSUtils.isNativeCDCLogFile(nativeCdcLogPathStr));
assertTrue(FSUtils.isCDCLogFile(nativeCdcLogPathStr));
}
@@ -185,6 +186,7 @@ public class TestHoodieLogFile {
assertEquals("cdc", hoodieLogFile.getFileExtension());
assertEquals("lance", hoodieLogFile.getSuffix());
assertTrue(hoodieLogFile.isCDC());
+ assertTrue(FSUtils.isNativeCDCLogFile(nativeCdcLogPathStr));
assertTrue(FSUtils.isCDCLogFile(nativeCdcLogPathStr));
}
@@ -205,6 +207,7 @@ public class TestHoodieLogFile {
assertEquals("1-0-1", hoodieLogFile.getLogWriteToken());
assertEquals("log", hoodieLogFile.getFileExtension());
assertEquals("custom", hoodieLogFile.getSuffix());
+ assertFalse(FSUtils.isNativeCDCLogFile(nativeLogPathStr));
assertFalse(FSUtils.isCDCLogFile(nativeLogPathStr));
}
@@ -235,7 +238,7 @@ public class TestHoodieLogFile {
logFiles.sort(HoodieLogFile.getLogFileComparator());
assertEquals(Arrays.asList(logFile, deleteFile, cdcFile), logFiles);
- assertFalse(FSUtils.matchNativeLogFile(nativeLogPath(LogExtensions.ARCHIVE_LOG_EXTENSION, 8)).isPresent());
+ assertFalse(FSUtils.isNativeLogFile(nativeLogPath(LogExtensions.ARCHIVE_LOG_EXTENSION, 8)));
}
private String nativeLogPath(String extension, int version) {
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestDefaultSerializer.java b/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestDefaultSerializer.java
index 42d53bd36985..ae828af87e48 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestDefaultSerializer.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestDefaultSerializer.java
@@ -41,7 +41,7 @@ class TestDefaultSerializer {
String fileId1 = UUID.randomUUID().toString();
HoodieInstant instant1 = new HoodieInstant(HoodieInstant.State.COMPLETED, "commit", "001", InstantComparatorV1.REQUESTED_TIME_BASED_COMPARATOR);
FileSlice fileSlice = new FileSlice(new HoodieFileGroupId(partition, fileId1), instant1.requestedTime(),
- new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName(instant1.requestedTime(), "1-0-1", fileId1, "parquet")), Collections.emptyList());
+ new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName(instant1.requestedTime(), "1-0-1", fileId1, ".parquet")), Collections.emptyList());
DefaultSerializer<FileSlice> serializer = new DefaultSerializer<>();
byte[] serializedValue = serializer.serialize(fileSlice);
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileGroupSerializer.java b/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileGroupSerializer.java
index 245af9fc6886..c6f4b29f396b 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileGroupSerializer.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileGroupSerializer.java
@@ -49,9 +49,9 @@ class TestHoodieFileGroupSerializer {
String fileId2 = UUID.randomUUID().toString();
HoodieInstant instant1 = new HoodieInstant(HoodieInstant.State.COMPLETED, "commit", "001", InstantComparatorV1.REQUESTED_TIME_BASED_COMPARATOR);
FileSlice fileSlice1 = new FileSlice(new HoodieFileGroupId(partition, fileId1), instant1.requestedTime(),
- new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName(instant1.requestedTime(), "1-0-1", fileId1, "parquet")), Collections.emptyList());
+ new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName(instant1.requestedTime(), "1-0-1", fileId1, ".parquet")), Collections.emptyList());
FileSlice fileSlice2 = new FileSlice(new HoodieFileGroupId(partition, fileId2), instant1.requestedTime(),
- new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName(instant1.requestedTime(), "1-0-1", fileId2, "parquet")), Collections.emptyList());
+ new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName(instant1.requestedTime(), "1-0-1", fileId2, ".parquet")), Collections.emptyList());
HoodieTimeline mockTimeline1 = new MockHoodieTimeline(Collections.singletonList(instant1));
HoodieFileGroup hoodieFileGroup1 = new HoodieFileGroup(partition, fileId1, mockTimeline1);
@@ -64,7 +64,7 @@ class TestHoodieFileGroupSerializer {
HoodieInstant instant2 = new HoodieInstant(HoodieInstant.State.COMPLETED, "commit", "002", InstantComparatorV1.REQUESTED_TIME_BASED_COMPARATOR);
HoodieTimeline mockTimeline2 = new MockHoodieTimeline(Collections.singletonList(instant2));
FileSlice fileSlice3 = new FileSlice(new HoodieFileGroupId(partition, fileId3), instant1.requestedTime(),
- new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName(instant2.requestedTime(), "1-0-1", fileId3, "parquet")), Collections.emptyList());
+ new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName(instant2.requestedTime(), "1-0-1", fileId3, ".parquet")), Collections.emptyList());
HoodieFileGroup hoodieFileGroup3 = new HoodieFileGroup(partition, fileId3, mockTimeline2);
hoodieFileGroup3.addFileSlice(fileSlice3);
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileSliceSerializer.java b/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileSliceSerializer.java
index 1e7e99a5a9c4..c6de830538df 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileSliceSerializer.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/serialization/TestHoodieFileSliceSerializer.java
@@ -34,27 +34,31 @@ import java.util.Arrays;
import java.util.List;
class TestHoodieFileSliceSerializer {
- private static final String LOG_FILE_PATH_FORMAT = "file:///tmp/basePath/partitionPath/.fileId%s_100.log.%s_1-0-1";
+ private static final String FILE_ID_1 = "b5068208-e1a4-11e6-bf01-fe55135034f3";
+ private static final String FILE_ID_2 = "b5068208-e1a4-11e6-bf01-fe55135034f4";
+ private static final String LOG_FILE_PATH_FORMAT = "file:///tmp/basePath/partitionPath/.%s_100.log.%s_1-0-1";
private static final short BLOCK_REPLICATION = 1;
- private static final StoragePath BASE_FILE_STORAGE_PATH_1 = new StoragePath("fileId-1/baseFilepath.parquet");
- private static final StoragePath BASE_FILE_STORAGE_PATH_2 = new StoragePath("fileId-1/baseFilepath.parquet");
- private static final StoragePath LOG_FILE_STORAGE_PATH_1 = new StoragePath(String.format(LOG_FILE_PATH_FORMAT, "1", "0"));
- private static final StoragePath LOG_FILE_STORAGE_PATH_2 = new StoragePath(String.format(LOG_FILE_PATH_FORMAT, "2", "2"));
+ private static final StoragePath BASE_FILE_STORAGE_PATH_1 =
+ new StoragePath("file:///tmp/basePath/partitionPath/" + FSUtils.makeBaseFileName("001", "1-0-1", FILE_ID_1, ".parquet"));
+ private static final StoragePath BASE_FILE_STORAGE_PATH_2 =
+ new StoragePath("file:///tmp/basePath/partitionPath/" + FSUtils.makeBaseFileName("001", "1-0-1", FILE_ID_2, ".parquet"));
+ private static final StoragePath LOG_FILE_STORAGE_PATH_1 = new StoragePath(String.format(LOG_FILE_PATH_FORMAT, FILE_ID_1, "0"));
+ private static final StoragePath LOG_FILE_STORAGE_PATH_2 = new StoragePath(String.format(LOG_FILE_PATH_FORMAT, FILE_ID_2, "2"));
@Test
void testSerDe() throws IOException {
HoodieFileSliceSerializer hoodieFileSliceSerializer = new HoodieFileSliceSerializer();
HoodieBaseFile baseFile1 = new HoodieBaseFile(new StoragePathInfo(BASE_FILE_STORAGE_PATH_1, 100, false, BLOCK_REPLICATION, 1024, 0));
HoodieLogFile logFile1 = new HoodieLogFile(new StoragePathInfo(LOG_FILE_STORAGE_PATH_1, 100, false, BLOCK_REPLICATION, 1024, 0));
- HoodieLogFile logFile2 = new HoodieLogFile("/dummy/base/" + FSUtils.makeLogFileName("fileId-1", HoodieLogFile.DELTA_EXTENSION, "001", 2, "1-0-1"));
+ HoodieLogFile logFile2 = new HoodieLogFile("/dummy/base/" + FSUtils.makeInlineLogFileName(FILE_ID_1, HoodieLogFile.DELTA_EXTENSION, "001", 2, "1-0-1"));
HoodieBaseFile baseFile2 = new HoodieBaseFile(new StoragePathInfo(BASE_FILE_STORAGE_PATH_2, 100, false, BLOCK_REPLICATION, 1024, 0));
HoodieLogFile logFile3 = new HoodieLogFile(new StoragePathInfo(LOG_FILE_STORAGE_PATH_2, 100, false, BLOCK_REPLICATION, 1024, 0));
- HoodieLogFile logFile4 = new HoodieLogFile("/dummy/base/" + FSUtils.makeLogFileName("fileId-2", HoodieLogFile.DELTA_EXTENSION, "002", 2, "1-0-1"));
+ HoodieLogFile logFile4 = new HoodieLogFile("/dummy/base/" + FSUtils.makeInlineLogFileName(FILE_ID_2, HoodieLogFile.DELTA_EXTENSION, "002", 2, "1-0-1"));
List<FileSlice> fileSliceList = Arrays.asList(
- new FileSlice(new HoodieFileGroupId("partition1", "fileId-1"), "001", baseFile1, Arrays.asList(logFile1, logFile2)),
- new FileSlice(new HoodieFileGroupId("partition2", "fileId-2"), "001", baseFile2, Arrays.asList(logFile3, logFile4)),
+ new FileSlice(new HoodieFileGroupId("partition1", FILE_ID_1), "001", baseFile1, Arrays.asList(logFile1, logFile2)),
+ new FileSlice(new HoodieFileGroupId("partition2", FILE_ID_2), "001", baseFile2, Arrays.asList(logFile3, logFile4)),
new FileSlice("partition3", "002", "fileId-3")
);
@@ -62,4 +66,4 @@ class TestHoodieFileSliceSerializer {
List<FileSlice> deserialized = hoodieFileSliceSerializer.deserialize(serializedBytes);
Assertions.assertEquals(fileSliceList, deserialized);
}
-}
\ No newline at end of file
+}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieFileGroupSizeEstimator.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieFileGroupSizeEstimator.java
index 33b66111531a..221e4ce75c28 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieFileGroupSizeEstimator.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieFileGroupSizeEstimator.java
@@ -50,7 +50,7 @@ class TestHoodieFileGroupSizeEstimator {
// setup mocks
HoodieFileGroupId fileGroupId = new HoodieFileGroupId("path1", UUID.randomUUID().toString());
List<FileSlice> fileSlices = Collections.singletonList(new FileSlice(fileGroupId, "001",
- new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName("001", "1-0-1", fileGroupId.getFileId(), "parquet")), Collections.emptyList()));
+ new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName("001", "1-0-1", fileGroupId.getFileId(), ".parquet")), Collections.emptyList()));
when(fileGroup1.getFileGroupId()).thenReturn(fileGroupId);
when(fileGroup1.getAllFileSlices()).thenReturn(fileSlices.stream()).thenReturn(fileSlices.stream());
@@ -71,7 +71,7 @@ class TestHoodieFileGroupSizeEstimator {
// setup mocks
HoodieFileGroupId fileGroupId = new HoodieFileGroupId("path1", UUID.randomUUID().toString());
List<FileSlice> fileSlices = IntStream.range(1, 100).mapToObj(i -> new FileSlice(fileGroupId, "001",
- new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName("00" + i, "1-0-1", fileGroupId.getFileId(), "parquet")), Collections.emptyList()))
+ new HoodieBaseFile("/tmp/" + FSUtils.makeBaseFileName("00" + i, "1-0-1", fileGroupId.getFileId(), ".parquet")), Collections.emptyList()))
.collect(Collectors.toList());
when(fileGroup1.getFileGroupId()).thenReturn(fileGroupId);
when(fileGroup1.getAllFileSlices()).thenReturn(fileSlices.stream()).thenReturn(fileSlices.stream());
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java
index b82846f5ef1f..79bf4b5da697 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java
@@ -89,7 +89,7 @@ public class TestPriorityBasedFileSystemView {
@BeforeEach
public void setUp() {
fsView = new PriorityBasedFileSystemView(primary, secondaryViewCreator, engineContext);
- testBaseFileStream = Stream.of(new HoodieBaseFile("test"));
+ testBaseFileStream = Stream.of(new HoodieBaseFile("test_1-0-1_001.parquet"));
testFileSliceStream = Stream.of(new FileSlice("2020-01-01", "20:20",
"file0001" + HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().getFileExtension()));
}
@@ -229,7 +229,7 @@ public class TestPriorityBasedFileSystemView {
@Test
public void testGetLatestBaseFile() {
Option<HoodieBaseFile> actual;
- Option<HoodieBaseFile> expected = Option.of(new HoodieBaseFile("test.file"));
+ Option<HoodieBaseFile> expected = Option.of(new HoodieBaseFile("test_1-0-1_001.parquet"));
String partitionPath = "/table2";
String fileID = "file.123";
@@ -260,7 +260,7 @@ public class TestPriorityBasedFileSystemView {
@Test
public void testGetBaseFileOn() {
Option<HoodieBaseFile> actual;
- Option<HoodieBaseFile> expected = Option.of(new HoodieBaseFile("test.file"));
+ Option<HoodieBaseFile> expected = Option.of(new HoodieBaseFile("test_1-0-1_001.parquet"));
String partitionPath = "/table2";
String instantTime = "2020-01-01";
String fileID = "file.123";
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
index 03be4951be1b..676fe95d3aa5 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
@@ -110,7 +110,7 @@ public class FileCreateUtils extends FileCreateUtilsBase {
}
public static String logFileMarkerFileName(String instantTime, String fileId, IOType ioType, String fileExtension, int logVersion) {
- return markerFileName(FSUtils.makeLogFileName(fileId, fileExtension, instantTime, logVersion, WRITE_TOKEN), ioType);
+ return markerFileName(FSUtils.makeInlineLogFileName(fileId, fileExtension, instantTime, logVersion, WRITE_TOKEN), ioType);
}
private static void deleteMetaFile(HoodieTableMetaClient metaClient, String instantTime, String suffix,
@@ -421,7 +421,7 @@ public class FileCreateUtils extends FileCreateUtilsBase {
public static String createMarkerFile(HoodieTableMetaClient metaClient, String partitionPath, String instantTime, String fileId, IOType ioType)
throws IOException {
if (IOType.APPEND == ioType) {
- String logFileName = FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, instantTime, HoodieLogFile.LOGFILE_BASE_VERSION, WRITE_TOKEN);
+ String logFileName = FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, instantTime, HoodieLogFile.LOGFILE_BASE_VERSION, WRITE_TOKEN);
String markerFileName = markerFileName(logFileName, ioType);
return createMarkerFile(metaClient, partitionPath, instantTime, markerFileName);
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsBase.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsBase.java
index d6df18e2f75e..d4721602a660 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsBase.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsBase.java
@@ -75,7 +75,7 @@ public class FileCreateUtilsBase {
public static String logFileName(String instantTime, String fileId, int version,
String fileExtension) {
- return FSUtils.makeLogFileName(fileId, fileExtension, instantTime, version, WRITE_TOKEN);
+ return FSUtils.makeInlineLogFileName(fileId, fileExtension, instantTime, version, WRITE_TOKEN);
}
public static String markerFileName(String fileName, IOType ioType) {
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsLegacy.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsLegacy.java
index e9cc3ddffe09..cf28d9fb2b78 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsLegacy.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsLegacy.java
@@ -229,7 +229,7 @@ public class FileCreateUtilsLegacy extends FileCreateUtilsBase {
public static String createMarkerFile(String basePath, String partitionPath, String instantTime, String fileId, IOType ioType)
throws IOException {
if (IOType.APPEND == ioType) {
- String logFileName = FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, instantTime, HoodieLogFile.LOGFILE_BASE_VERSION, WRITE_TOKEN);
+ String logFileName = FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, instantTime, HoodieLogFile.LOGFILE_BASE_VERSION, WRITE_TOKEN);
String markerFileName = markerFileName(logFileName, ioType);
return createMarkerFile(basePath, partitionPath, instantTime, markerFileName);
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
index 7df2f23096ce..bc7638d895b3 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
@@ -161,7 +161,7 @@ public class HoodieTestUtils {
public static List<StoragePath> listNativeLogFiles(HoodieStorage storage, String basePath) throws IOException {
return storage.listFiles(new StoragePath(basePath)).stream()
.map(StoragePathInfo::getPath)
- .filter(path -> FSUtils.matchNativeLogFile(path.getName()).isPresent())
+ .filter(path -> FSUtils.isNativeLogFile(path.getName()))
.collect(Collectors.toList());
}
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FlinkRowDataReaderContext.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FlinkRowDataReaderContext.java
index b60340550c74..bd5b05e1d163 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FlinkRowDataReaderContext.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/FlinkRowDataReaderContext.java
@@ -101,12 +101,13 @@ public class FlinkRowDataReaderContext extends HoodieReaderContext<RowData> {
HoodieSchema dataSchema,
HoodieSchema requiredSchema,
HoodieStorage storage) throws IOException {
+ boolean isLogFile = FSUtils.isLogFile(filePath);
// disable schema evolution in fileReader if it's log file, since schema evolution for log file is handled in `FileGroupRecordBuffer`
- InternalSchemaManager schemaManager = FSUtils.isLogFile(filePath) ? InternalSchemaManager.DISABLED : internalSchemaManager.get();
+ InternalSchemaManager schemaManager = isLogFile ? InternalSchemaManager.DISABLED : internalSchemaManager.get();
// Log files only reach this method for parquet data blocks; base files are resolved by their extension.
// Format-specific handling lives in the readers themselves, so this method stays format-agnostic.
- boolean isInlineLogFile = FSUtils.isLogFile(filePath) && FSUtils.matchNativeLogFile(filePath.getName()).isEmpty();
+ boolean isInlineLogFile = isLogFile && FSUtils.isInlineLogFile(filePath.getName());
HoodieFileFormat format = isInlineLogFile ? HoodieFileFormat.PARQUET : HoodieFileFormat.fromFileExtension(filePath.getFileExtension());
HoodieRowDataFileReader reader = (HoodieRowDataFileReader) HoodieIOFactory.getIOFactory(storage)
.getReaderFactory(HoodieRecord.HoodieRecordType.FLINK)
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcIterators.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcIterators.java
index cbd0083f81a5..a4a0c33d6eca 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcIterators.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/cdc/CdcIterators.java
@@ -526,9 +526,9 @@ public final class CdcIterators {
}
private static boolean isNativeCdcFileSplit(HoodieCDCFileSplit fileSplit) {
- boolean nativeCdc = FSUtils.matchNativeLogFile(fileSplit.getCdcFiles().get(0)).isPresent();
+ boolean nativeCdc = FSUtils.isNativeLogFile(fileSplit.getCdcFiles().get(0));
ValidationUtils.checkState(fileSplit.getCdcFiles().stream()
- .allMatch(path -> FSUtils.matchNativeLogFile(path).isPresent() == nativeCdc),
+ .allMatch(path -> FSUtils.isNativeLogFile(path) == nativeCdc),
"CDC file split cannot mix inline and native CDC log files");
return nativeCdc;
}
@@ -789,9 +789,11 @@ public final class CdcIterators {
}
public static MergeOnReadInputSplit singleLogFile2Split(String tablePath, String filePath, long maxCompactionMemoryInBytes) {
+ StoragePath logPath = new StoragePath(filePath);
+ HoodieLogFile logFile = new HoodieLogFile(logPath);
return new MergeOnReadInputSplit(0, null, Option.of(Collections.singletonList(filePath)),
- FSUtils.getDeltaCommitTimeFromLogPath(new StoragePath(filePath)), tablePath, maxCompactionMemoryInBytes,
- FlinkOptions.REALTIME_PAYLOAD_COMBINE, null, FSUtils.getFileIdFromLogPath(new StoragePath(filePath)),
- FSUtils.getRelativePartitionPath(new StoragePath(tablePath), new StoragePath(filePath).getParent()));
+ logFile.getDeltaCommitTime(), tablePath, maxCompactionMemoryInBytes,
+ FlinkOptions.REALTIME_PAYLOAD_COMBINE, null, logFile.getFileId(),
+ FSUtils.getRelativePartitionPath(new StoragePath(tablePath), logPath.getParent()));
}
}
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java
index 7a5556002602..7a3a5c2a9567 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java
@@ -19,6 +19,7 @@
package org.apache.hudi.common.fs;
import org.apache.hudi.common.engine.HoodieLocalEngineContext;
+import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.cdc.HoodieCDCUtils;
@@ -26,9 +27,11 @@ import org.apache.hudi.common.table.timeline.TimelineUtils;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.CollectionUtils;
+import org.apache.hudi.common.util.ExternalFilePathUtil;
import org.apache.hudi.common.util.HoodieStorageUtils;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.exception.HoodieValidationException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.hadoop.fs.HoodieWrapperFileSystem;
import org.apache.hudi.hadoop.fs.inline.HadoopInLineFSUtils;
@@ -173,8 +176,13 @@ public class TestFSUtils extends HoodieCommonTestHarness {
String fullFileName = FSUtils.makeBaseFileName(instantTime, TEST_WRITE_TOKEN, fileName, HoodieCommonTestHarness.BASE_FILE_EXTENSION);
assertEquals(instantTime, FSUtils.getCommitTime(fullFileName));
// test log file name
- fullFileName = FSUtils.makeLogFileName(fileName, HOODIE_LOG.getFileExtension(), instantTime, 1, TEST_WRITE_TOKEN);
+ fullFileName = FSUtils.makeInlineLogFileName(fileName, HOODIE_LOG.getFileExtension(), instantTime, 1, TEST_WRITE_TOKEN);
assertEquals(instantTime, FSUtils.getCommitTime(fullFileName));
+
+ assertEquals(instantTime, FSUtils.getCommitTime(fileName + "_" + TEST_WRITE_TOKEN + "_" + instantTime));
+
+ String externalFileName = ExternalFilePathUtil.appendCommitTimeAndExternalFileMarker("file_1.parquet", instantTime);
+ assertEquals(instantTime, FSUtils.getCommitTime(externalFileName));
}
@Test
@@ -185,6 +193,75 @@ public class TestFSUtils extends HoodieCommonTestHarness {
assertEquals(fileName, FSUtils.getFileId(fullFileName));
}
+ @Test
+ public void testIsBaseFile() {
+ String baseFileName = FSUtils.makeBaseFileName("20240706120100123", TEST_WRITE_TOKEN, UUID.randomUUID().toString(), ".parquet");
+ assertTrue(FSUtils.isBaseFile(baseFileName));
+ assertTrue(FSUtils.isBaseFile("foo_20240101.parquet"));
+ assertTrue(FSUtils.isBaseFile("a_b_xyz.parquet"));
+ assertTrue(FSUtils.isBaseFile(ExternalFilePathUtil.appendCommitTimeAndExternalFileMarker("file_1.parquet", "20240706120100123")));
+ }
+
+ @Test
+ public void testFileNameParser() {
+ String fileId = UUID.randomUUID().toString();
+ String baseFileName = FSUtils.makeBaseFileName("20240706120100123", TEST_WRITE_TOKEN, fileId, ".parquet");
+ FileNameParser.BaseFileName parsedBaseFileName = FileNameParser.parseBaseFile(baseFileName).get();
+ assertEquals(fileId, parsedBaseFileName.getFileId());
+ assertEquals(TEST_WRITE_TOKEN, parsedBaseFileName.getWriteToken());
+ assertEquals("20240706120100123", parsedBaseFileName.getCommitTime());
+ assertEquals(".parquet", parsedBaseFileName.getFileExtension());
+
+ FileNameParser.BaseFileName parsedBaseFileNameWithoutExtension =
+ FileNameParser.parseBaseFile(fileId + "_" + TEST_WRITE_TOKEN + "_20240706120100123").get();
+ assertEquals(fileId, parsedBaseFileNameWithoutExtension.getFileId());
+ assertEquals(TEST_WRITE_TOKEN, parsedBaseFileNameWithoutExtension.getWriteToken());
+ assertEquals("20240706120100123", parsedBaseFileNameWithoutExtension.getCommitTime());
+ assertEquals("", parsedBaseFileNameWithoutExtension.getFileExtension());
+
+ FileNameParser.BaseFileName parsedLegacyBaseFileName = FileNameParser.parseBaseFile("file_1.parquet").get();
+ assertEquals("file", parsedLegacyBaseFileName.getFileId());
+ assertEquals("", parsedLegacyBaseFileName.getWriteToken());
+ assertEquals("1", parsedLegacyBaseFileName.getCommitTime());
+ assertEquals(".parquet", parsedLegacyBaseFileName.getFileExtension());
+
+ FileNameParser.BaseFileName parsedExternalBaseFileName = FileNameParser.parseBaseFile(
+ ExternalFilePathUtil.appendCommitTimeAndExternalFileMarker("file_1.parquet", "20240706120100123")).get();
+ assertEquals("file_1.parquet", parsedExternalBaseFileName.getFileId());
+ assertEquals("", parsedExternalBaseFileName.getWriteToken());
+ assertEquals("20240706120100123", parsedExternalBaseFileName.getCommitTime());
+ assertEquals(".parquet", parsedExternalBaseFileName.getFileExtension());
+
+ String logFileName = FSUtils.makeInlineLogFileName(fileId, ".log", "20240706120100123", 2, TEST_WRITE_TOKEN)
+ + HoodieCDCUtils.CDC_LOGFILE_SUFFIX;
+ FileNameParser.LogFileName parsedLogFileName = FileNameParser.parseLogFile(logFileName).get();
+ assertEquals(fileId, parsedLogFileName.getFileId());
+ assertEquals("20240706120100123", parsedLogFileName.getDeltaCommitTime());
+ assertEquals(2, parsedLogFileName.getLogVersion());
+ assertEquals(TEST_WRITE_TOKEN, parsedLogFileName.getWriteToken());
+ assertEquals(1, parsedLogFileName.getTaskPartitionId());
+ assertEquals(0, parsedLogFileName.getStageId());
+ assertEquals(1, parsedLogFileName.getTaskAttemptId());
+ assertEquals("log", parsedLogFileName.getFileExtension());
+ assertEquals(HoodieCDCUtils.CDC_LOGFILE_SUFFIX, parsedLogFileName.getSuffix());
+ assertTrue(parsedLogFileName.isCDCLogFile());
+ assertFalse(parsedLogFileName.isNativeLogFile());
+
+ String nativeLogFileName = FSUtils.makeNativeLogFileName(fileId, TEST_WRITE_TOKEN, "20240706120100123",
+ 3, "deletes", HoodieFileFormat.PARQUET);
+ FileNameParser.LogFileName parsedNativeLogFileName = FileNameParser.parseLogFile(nativeLogFileName).get();
+ assertEquals(fileId, parsedNativeLogFileName.getFileId());
+ assertEquals("20240706120100123", parsedNativeLogFileName.getDeltaCommitTime());
+ assertEquals(3, parsedNativeLogFileName.getLogVersion());
+ assertEquals(TEST_WRITE_TOKEN, parsedNativeLogFileName.getWriteToken());
+ assertEquals("deletes", parsedNativeLogFileName.getFileExtension());
+ assertEquals("parquet", parsedNativeLogFileName.getSuffix());
+ assertTrue(parsedNativeLogFileName.isNativeLogFile());
+
+ assertEquals(fileId, FileNameParser.getFileIdPfxFromFileId(fileId + "-4"));
+ assertThrows(HoodieValidationException.class, () -> FileNameParser.getFileIdPfxFromFileId("fileIdWithoutNumericSuffix"));
+ }
+
@Test
public void testEnvVarVariablesPickedup() {
environmentVariables.set("HOODIE_ENV_fs_DOT_key1", "value1");
@@ -244,7 +321,7 @@ public class TestFSUtils extends HoodieCommonTestHarness {
// Check if log file names are parsable by FSUtils method
String partitionPath = "2019/01/01/";
String fileName = UUID.randomUUID().toString();
- String logFile = FSUtils.makeLogFileName(fileName, ".log", "100", 2, "1-0-1");
+ String logFile = FSUtils.makeInlineLogFileName(fileName, ".log", "100", 2, "1-0-1");
StoragePath rlPath = new StoragePath(new StoragePath(partitionPath), logFile);
StoragePath inlineFsPath = HadoopInLineFSUtils.getInlineFilePath(
new StoragePath(rlPath.toUri()), "file", 0, 100);
@@ -266,7 +343,7 @@ public class TestFSUtils extends HoodieCommonTestHarness {
public void testCdcLogFileName() {
String partitionPath = "2022/11/04/";
String fileName = UUID.randomUUID().toString();
- String logFile = FSUtils.makeLogFileName(fileName, ".log", "100", 2, "1-0-1") + HoodieCDCUtils.CDC_LOGFILE_SUFFIX;
+ String logFile = FSUtils.makeInlineLogFileName(fileName, ".log", "100", 2, "1-0-1") + HoodieCDCUtils.CDC_LOGFILE_SUFFIX;
StoragePath path = new StoragePath(new StoragePath(partitionPath), logFile);
assertTrue(FSUtils.isLogFile(path));
@@ -305,7 +382,7 @@ public class TestFSUtils extends HoodieCommonTestHarness {
public void testArchiveLogFileName() {
String partitionPath = "2022/11/04/";
String fileName = "commits";
- String logFile = FSUtils.makeLogFileName(fileName, ".archive", "", 2, "1-0-1");
+ String logFile = FSUtils.makeInlineLogFileName(fileName, ".archive", "", 2, "1-0-1");
StoragePath path = new StoragePath(new StoragePath(partitionPath), logFile);
assertFalse(FSUtils.isLogFile(path));
@@ -339,12 +416,12 @@ public class TestFSUtils extends HoodieCommonTestHarness {
*/
@Test
public void testLogFilesComparison() {
- String log1Ver0W0 = FSUtils.makeLogFileName("file1", ".log", "1", 0, "0-0-1");
- String log1Ver0W1 = FSUtils.makeLogFileName("file1", ".log", "1", 0, "1-1-1");
- String log1Ver1W0 = FSUtils.makeLogFileName("file1", ".log", "1", 1, "0-0-1");
- String log1Ver1W1 = FSUtils.makeLogFileName("file1", ".log", "1", 1, "1-1-1");
- String log1base2W0 = FSUtils.makeLogFileName("file1", ".log", "2", 0, "0-0-1");
- String log1base2W1 = FSUtils.makeLogFileName("file1", ".log", "2", 0, "1-1-1");
+ String log1Ver0W0 = FSUtils.makeInlineLogFileName("file1", ".log", "1", 0, "0-0-1");
+ String log1Ver0W1 = FSUtils.makeInlineLogFileName("file1", ".log", "1", 0, "1-1-1");
+ String log1Ver1W0 = FSUtils.makeInlineLogFileName("file1", ".log", "1", 1, "0-0-1");
+ String log1Ver1W1 = FSUtils.makeInlineLogFileName("file1", ".log", "1", 1, "1-1-1");
+ String log1base2W0 = FSUtils.makeInlineLogFileName("file1", ".log", "2", 0, "0-0-1");
+ String log1base2W1 = FSUtils.makeInlineLogFileName("file1", ".log", "2", 0, "1-1-1");
List<HoodieLogFile> logFiles =
Stream.of(log1Ver1W1, log1base2W0, log1base2W1, log1Ver1W0, log1Ver0W1, log1Ver0W0)
@@ -359,11 +436,11 @@ public class TestFSUtils extends HoodieCommonTestHarness {
@Test
public void testLogFilesComparisonWithCDCFile() {
- HoodieLogFile log1 = new HoodieLogFile(new StoragePath(FSUtils.makeLogFileName("file1", ".log", "1", 0, "0-0-1")));
- HoodieLogFile log2 = new HoodieLogFile(new StoragePath(FSUtils.makeLogFileName("file1", ".log", "2", 0, "0-0-1")));
- HoodieLogFile log3 = new HoodieLogFile(new StoragePath(FSUtils.makeLogFileName("file1", ".log", "2", 1, "0-0-1")));
- HoodieLogFile log4 = new HoodieLogFile(new StoragePath(FSUtils.makeLogFileName("file1", ".log", "2", 1, "1-1-1")));
- HoodieLogFile log5 = new HoodieLogFile(new StoragePath(FSUtils.makeLogFileName("file1", ".log", "2", 1, "1-1-1") + HoodieCDCUtils.CDC_LOGFILE_SUFFIX));
+ HoodieLogFile log1 = new HoodieLogFile(new StoragePath(FSUtils.makeInlineLogFileName("file1", ".log", "1", 0, "0-0-1")));
+ HoodieLogFile log2 = new HoodieLogFile(new StoragePath(FSUtils.makeInlineLogFileName("file1", ".log", "2", 0, "0-0-1")));
+ HoodieLogFile log3 = new HoodieLogFile(new StoragePath(FSUtils.makeInlineLogFileName("file1", ".log", "2", 1, "0-0-1")));
+ HoodieLogFile log4 = new HoodieLogFile(new StoragePath(FSUtils.makeInlineLogFileName("file1", ".log", "2", 1, "1-1-1")));
+ HoodieLogFile log5 = new HoodieLogFile(new StoragePath(FSUtils.makeInlineLogFileName("file1", ".log", "2", 1, "1-1-1") + HoodieCDCUtils.CDC_LOGFILE_SUFFIX));
TreeSet<HoodieLogFile> logFilesSet = new TreeSet<>(HoodieLogFile.getLogFileComparator());
logFilesSet.add(log1);
@@ -400,7 +477,7 @@ public class TestFSUtils extends HoodieCommonTestHarness {
assertEquals(instantTime, FSUtils.getCommitTime(dataFileName));
assertEquals(fileId, FSUtils.getFileId(dataFileName));
- String logFileName = FSUtils.makeLogFileName(fileId, LOG_EXTENSION, instantTime, version, writeToken);
+ String logFileName = FSUtils.makeInlineLogFileName(fileId, LOG_EXTENSION, instantTime, version, writeToken);
assertTrue(FSUtils.isLogFile(new StoragePath(logFileName)));
assertEquals(instantTime, FSUtils.getDeltaCommitTimeFromLogPath(new StoragePath(logFileName)));
assertEquals(fileId, FSUtils.getFileIdFromLogPath(new StoragePath(logFileName)));
@@ -410,11 +487,11 @@ public class TestFSUtils extends HoodieCommonTestHarness {
// create three versions of log file
java.nio.file.Path partitionPath = Paths.get(basePath, partitionStr);
Files.createDirectories(partitionPath);
- String log1 = FSUtils.makeLogFileName(fileId, LOG_EXTENSION, instantTime, 1, writeToken);
+ String log1 = FSUtils.makeInlineLogFileName(fileId, LOG_EXTENSION, instantTime, 1, writeToken);
Files.createFile(partitionPath.resolve(log1));
- String log2 = FSUtils.makeLogFileName(fileId, LOG_EXTENSION, instantTime, 2, writeToken);
+ String log2 = FSUtils.makeInlineLogFileName(fileId, LOG_EXTENSION, instantTime, 2, writeToken);
Files.createFile(partitionPath.resolve(log2));
- String log3 = FSUtils.makeLogFileName(fileId, LOG_EXTENSION, instantTime, 3, writeToken);
+ String log3 = FSUtils.makeInlineLogFileName(fileId, LOG_EXTENSION, instantTime, 3, writeToken);
Files.createFile(partitionPath.resolve(log3));
assertEquals(3, (int) FSUtils.getLatestLogVersion(
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/model/TestHoodieFileGroup.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/model/TestHoodieFileGroup.java
index ddd742a0d757..0e37280adfb9 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/model/TestHoodieFileGroup.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/model/TestHoodieFileGroup.java
@@ -62,7 +62,7 @@ public class TestHoodieFileGroup {
HoodieFileGroup fileGroup = new HoodieFileGroup("", "data",
activeTimeline.getCommitsTimeline().filterCompletedInstants());
for (int i = 0; i < 3; i++) {
- HoodieBaseFile baseFile = new HoodieBaseFile("data_1_00" + i);
+ HoodieBaseFile baseFile = new HoodieBaseFile("data_1_00" + i + ".parquet");
fileGroup.addBaseFile(baseFile);
}
assertEquals(2, fileGroup.getAllFileSlices().count());
@@ -84,7 +84,7 @@ public class TestHoodieFileGroup {
HoodieFileGroup fileGroup = new HoodieFileGroup("", "data",
activeTimeline.getCommitsTimeline().filterCompletedInstants());
for (int i = 0; i < 3; i++) {
- HoodieBaseFile baseFile = new HoodieBaseFile("data_1_00" + i);
+ HoodieBaseFile baseFile = new HoodieBaseFile("data_1_00" + i + ".parquet");
fileGroup.addBaseFile(baseFile);
fileGroup.addLogFile(queryView, new HoodieLogFile(new StoragePath(FileCreateUtilsLegacy.logFileName(preTableVersion8 ? "001" : "00" + i, "data", i))));
}
@@ -105,7 +105,7 @@ public class TestHoodieFileGroup {
).collect(Collectors.toList()));
HoodieFileGroup fileGroup = new HoodieFileGroup("", "data", activeTimeline.filterCompletedAndCompactionInstants());
for (int i = 0; i < 7; i++) {
- HoodieBaseFile baseFile = new HoodieBaseFile("data_1_0" + i);
+ HoodieBaseFile baseFile = new HoodieBaseFile("data_1_0" + i + ".parquet");
fileGroup.addBaseFile(baseFile);
}
List<FileSlice> allFileSlices = fileGroup.getAllFileSlices().collect(Collectors.toList());
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java
index 4f11960cc082..c3b4add07e15 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java
@@ -206,11 +206,11 @@ class TestTableSchemaResolver {
HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
// create 3 base files and 2 log files
HoodieWriteStat baseFileWriteStat = buildWriteStat("partition1/baseFile1.parquet", 10, 100);
- HoodieWriteStat logFileWriteStat = buildWriteStat("partition1/" + FSUtils.makeLogFileName("001", ".log", "100", 2, "1-0-1"), 0, 10);
+ HoodieWriteStat logFileWriteStat = buildWriteStat("partition1/" + FSUtils.makeInlineLogFileName("001", ".log", "100", 2, "1-0-1"), 0, 10);
// we don't expect any interactions with this write stat since the code should exit as soon as a valid schema is found
HoodieWriteStat baseFileWriteStat2 = mock(HoodieWriteStat.class);
// files that only have deletes will be skipped
- HoodieWriteStat logFileWithOnlyDeletes = buildWriteStat("partition1/" + FSUtils.makeLogFileName("002", ".log", "100", 2, "1-0-1"), 0, 0);
+ HoodieWriteStat logFileWithOnlyDeletes = buildWriteStat("partition1/" + FSUtils.makeInlineLogFileName("002", ".log", "100", 2, "1-0-1"), 0, 0);
HoodieWriteStat baseFileWithOnlyDeletes = buildWriteStat("partition2/baseFile2.parquet", 0, 0);
commitMetadata.addWriteStat("partition1", baseFileWriteStat);
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java
index 3afc3abd4dde..ac068ec15a65 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java
@@ -249,9 +249,9 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
String deltaInstantTime3 = "3";
String fileName1 =
- FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, deltaInstantTime2, 0, TEST_WRITE_TOKEN);
+ FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, deltaInstantTime2, 0, TEST_WRITE_TOKEN);
String fileName2 =
- FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? deltaInstantTime2 : deltaInstantTime3, 1, TEST_WRITE_TOKEN);
+ FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? deltaInstantTime2 : deltaInstantTime3, 1, TEST_WRITE_TOKEN);
Paths.get(basePath, partitionPath, fileName1).toFile().createNewFile();
Paths.get(basePath, partitionPath, fileName2).toFile().createNewFile();
@@ -508,9 +508,9 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
String deltaInstantTime1 = "2";
String deltaInstantTime2 = "3";
String fileName1 =
- FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, deltaInstantTime1, 0, TEST_WRITE_TOKEN);
+ FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, deltaInstantTime1, 0, TEST_WRITE_TOKEN);
String fileName2 =
- FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? deltaInstantTime1 : deltaInstantTime2, 1, TEST_WRITE_TOKEN);
+ FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? deltaInstantTime1 : deltaInstantTime2, 1, TEST_WRITE_TOKEN);
// create a dummy log file mimicing cloud stores marker files
String fileName3 = "_GCS_SYNCABLE_TEMPFILE_" + fileName1;
String fileName4 = "_DUMMY_" + fileName1.substring(1);
@@ -583,13 +583,13 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
String baseFile1 = FSUtils.makeBaseFileName(instantTime1, TEST_WRITE_TOKEN, fileId, BASE_FILE_EXTENSION);
String deltaFile1 =
- FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, deltaInstantTime1, 0,
+ FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, deltaInstantTime1, 0,
TEST_WRITE_TOKEN);
String deltaFile2 =
- FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, deltaInstantTime2, 0,
+ FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, deltaInstantTime2, 0,
TEST_WRITE_TOKEN);
String deltaFile3 =
- FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, deltaInstantTime3, 0,
+ FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, deltaInstantTime3, 0,
TEST_WRITE_TOKEN);
Paths.get(basePath, partitionPath, baseFile1).toFile().createNewFile();
@@ -710,7 +710,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
String instantTime1 = "1";
String fileName1 =
- FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, instantTime1, 0, TEST_WRITE_TOKEN);
+ FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, instantTime1, 0, TEST_WRITE_TOKEN);
Paths.get(basePath, partitionPath, fileName1).toFile().createNewFile();
HoodieActiveTimeline commitTimeline = metaClient.getActiveTimeline();
@@ -737,9 +737,9 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
String fileId2 = UUID.randomUUID().toString();
String instantTime1 = "1";
String fileName1 =
- FSUtils.makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, instantTime1, 0, TEST_WRITE_TOKEN);
+ FSUtils.makeInlineLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, instantTime1, 0, TEST_WRITE_TOKEN);
String fileName2 =
- FSUtils.makeLogFileName(fileId2, HoodieLogFile.DELTA_EXTENSION, instantTime1, 0, TEST_WRITE_TOKEN);
+ FSUtils.makeInlineLogFileName(fileId2, HoodieLogFile.DELTA_EXTENSION, instantTime1, 0, TEST_WRITE_TOKEN);
Paths.get(basePath, partitionPath1, fileName1).toFile().createNewFile();
Paths.get(basePath, partitionPath2, fileName2).toFile().createNewFile();
@@ -813,9 +813,9 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
String dataFileName = FSUtils.makeBaseFileName(instantTime1, TEST_WRITE_TOKEN, fileId, BASE_FILE_EXTENSION);
new File(basePath + "/" + partitionPath + "/" + dataFileName).createNewFile();
String fileName1 =
- FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? instantTime1 : deltaInstantTime1, 0, TEST_WRITE_TOKEN);
+ FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? instantTime1 : deltaInstantTime1, 0, TEST_WRITE_TOKEN);
String fileName2 =
- FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? instantTime1 : deltaInstantTime2, 1, TEST_WRITE_TOKEN);
+ FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? instantTime1 : deltaInstantTime2, 1, TEST_WRITE_TOKEN);
new File(basePath + "/" + partitionPath + "/" + fileName1).createNewFile();
new File(basePath + "/" + partitionPath + "/" + fileName2).createNewFile();
HoodieActiveTimeline commitTimeline = metaClient.getActiveTimeline();
@@ -914,8 +914,8 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
}
String logTime1 = preTableVersion8 ? (skipCreatingDataFile ? deltaInstantTime1 : instantTime1) : deltaInstantTime1;
String logTime2 = preTableVersion8 ? (skipCreatingDataFile ? deltaInstantTime1 : instantTime1) : deltaInstantTime2;
- String fileName1 = FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, logTime1, 0, TEST_WRITE_TOKEN);
- String fileName2 = FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, logTime2, 1, TEST_WRITE_TOKEN);
+ String fileName1 = FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, logTime1, 0, TEST_WRITE_TOKEN);
+ String fileName2 = FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, logTime2, 1, TEST_WRITE_TOKEN);
new File(basePath + "/" + partitionPath + "/" + fileName1).createNewFile();
new File(basePath + "/" + partitionPath + "/" + fileName2).createNewFile();
HoodieActiveTimeline commitTimeline = metaClient.getActiveTimeline();
@@ -992,9 +992,9 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
String logTime4 = preTableVersion8 ? compactionRequestedTime : deltaInstantTime4;
String logTime5 = preTableVersion8 ? compactionRequestedTime : deltaInstantTime5;
String fileName3 =
- FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, logTime4, preTableVersion8 ? 2 : 0, TEST_WRITE_TOKEN);
+ FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, logTime4, preTableVersion8 ? 2 : 0, TEST_WRITE_TOKEN);
String fileName4 =
- FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, logTime5, preTableVersion8 ? 3 : 1, TEST_WRITE_TOKEN);
+ FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, logTime5, preTableVersion8 ? 3 : 1, TEST_WRITE_TOKEN);
new File(basePath + "/" + partitionPath + "/" + fileName3).createNewFile();
new File(basePath + "/" + partitionPath + "/" + fileName4).createNewFile();
HoodieInstant deltaInstant4 = INSTANT_GENERATOR.createNewInstant(State.INFLIGHT, HoodieTimeline.DELTA_COMMIT_ACTION, deltaInstantTime4);
@@ -1098,13 +1098,13 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
String orphanDataFileName = FSUtils.makeBaseFileName(invalidInstantId, TEST_WRITE_TOKEN, orphanFileId1, BASE_FILE_EXTENSION);
new File(basePath + "/" + partitionPath + "/" + orphanDataFileName).createNewFile();
String orphanLogFileName =
- FSUtils.makeLogFileName(orphanFileId2, HoodieLogFile.DELTA_EXTENSION, invalidInstantId, 0,
+ FSUtils.makeInlineLogFileName(orphanFileId2, HoodieLogFile.DELTA_EXTENSION, invalidInstantId, 0,
TEST_WRITE_TOKEN);
new File(basePath + "/" + partitionPath + "/" + orphanLogFileName).createNewFile();
String inflightDataFileName = FSUtils.makeBaseFileName(inflightDeltaInstantTime, TEST_WRITE_TOKEN, inflightFileId1, BASE_FILE_EXTENSION);
new File(basePath + "/" + partitionPath + "/" + inflightDataFileName).createNewFile();
String inflightLogFileName =
- FSUtils.makeLogFileName(inflightFileId2, HoodieLogFile.DELTA_EXTENSION,
+ FSUtils.makeInlineLogFileName(inflightFileId2, HoodieLogFile.DELTA_EXTENSION,
inflightDeltaInstantTime, 0, TEST_WRITE_TOKEN);
new File(basePath + "/" + partitionPath + "/" + inflightLogFileName).createNewFile();
// Mark instant as inflight
@@ -1319,10 +1319,10 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
new File(fullPartitionPath + FSUtils.makeBaseFileName(commitTime2, TEST_WRITE_TOKEN, fileId1, BASE_FILE_EXTENSION)).createNewFile();
new File(fullPartitionPath + FSUtils.makeBaseFileName(commitTime3, TEST_WRITE_TOKEN, fileId1, BASE_FILE_EXTENSION)).createNewFile();
new File(fullPartitionPath
- + FSUtils.makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime3 : commitTime4, 0, TEST_WRITE_TOKEN))
+ + FSUtils.makeInlineLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime3 : commitTime4, 0, TEST_WRITE_TOKEN))
.createNewFile();
new File(fullPartitionPath
- + FSUtils.makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime3 : commitTime5, 1, TEST_WRITE_TOKEN))
+ + FSUtils.makeInlineLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime3 : commitTime5, 1, TEST_WRITE_TOKEN))
.createNewFile();
// file group 2
@@ -1330,14 +1330,14 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
new File(fullPartitionPath + FSUtils.makeBaseFileName(commitTime3, TEST_WRITE_TOKEN, fileId2, BASE_FILE_EXTENSION)).createNewFile();
new File(fullPartitionPath + FSUtils.makeBaseFileName(commitTime4, TEST_WRITE_TOKEN, fileId2, BASE_FILE_EXTENSION)).createNewFile();
new File(fullPartitionPath
- + FSUtils.makeLogFileName(fileId2, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime4 : commitTime5, 0, TEST_WRITE_TOKEN))
+ + FSUtils.makeInlineLogFileName(fileId2, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime4 : commitTime5, 0, TEST_WRITE_TOKEN))
.createNewFile();
// file group 3
new File(fullPartitionPath + FSUtils.makeBaseFileName(commitTime3, TEST_WRITE_TOKEN, fileId3, BASE_FILE_EXTENSION)).createNewFile();
new File(fullPartitionPath + FSUtils.makeBaseFileName(commitTime4, TEST_WRITE_TOKEN, fileId3, BASE_FILE_EXTENSION)).createNewFile();
new File(fullPartitionPath
- + FSUtils.makeLogFileName(fileId4, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime4 : commitTime5, 0, TEST_WRITE_TOKEN))
+ + FSUtils.makeInlineLogFileName(fileId4, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime4 : commitTime5, 0, TEST_WRITE_TOKEN))
.createNewFile();
// Create commit/clean files
@@ -1405,19 +1405,19 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
for (HoodieLogFile logFile : logFilesList) {
filenames.add(logFile.getFileName());
}
- String l = FSUtils.makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime3 : commitTime5, 0,
+ String l = FSUtils.makeInlineLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime3 : commitTime5, 0,
TEST_WRITE_TOKEN);
assertTrue(filenames
- .contains(FSUtils.makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime3 : commitTime4, 0,
+ .contains(FSUtils.makeInlineLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime3 : commitTime4, 0,
TEST_WRITE_TOKEN)));
assertTrue(filenames
- .contains(FSUtils.makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime3 : commitTime5, 1,
+ .contains(FSUtils.makeInlineLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime3 : commitTime5, 1,
TEST_WRITE_TOKEN)));
assertTrue(filenames
- .contains(FSUtils.makeLogFileName(fileId2, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime4 : commitTime5, 0,
+ .contains(FSUtils.makeInlineLogFileName(fileId2, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime4 : commitTime5, 0,
TEST_WRITE_TOKEN)));
assertTrue(filenames
- .contains(FSUtils.makeLogFileName(fileId4, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime4 : commitTime5, 0,
+ .contains(FSUtils.makeInlineLogFileName(fileId4, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime4 : commitTime5, 0,
TEST_WRITE_TOKEN)));
// Reset the max commit time
@@ -1440,7 +1440,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
List<String> logFilesNames = rtView.getLatestFileSlicesBeforeOrOn("2016/05/01", commitTime4, true)
.flatMap(FileSlice::getLogFiles).map(HoodieLogFile::getFileName).collect(Collectors.toList());
assertEquals(preTableVersion8 ? 4 : 3, logFilesNames.size());
- assertTrue(logFilesNames.contains(FSUtils.makeLogFileName(fileId2, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime4 : commitTime5, 0, TEST_WRITE_TOKEN)));
+ assertTrue(logFilesNames.contains(FSUtils.makeInlineLogFileName(fileId2, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime4 : commitTime5, 0, TEST_WRITE_TOKEN)));
}
@Test
@@ -1531,7 +1531,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
// file group A
new File(fullPartitionPath + FSUtils.makeBaseFileName(commitTime1, TEST_WRITE_TOKEN, fileIdA, BASE_FILE_EXTENSION)).createNewFile();
new File(fullPartitionPath
- + FSUtils.makeLogFileName(fileIdA, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime1 : commitTime2, 0, TEST_WRITE_TOKEN))
+ + FSUtils.makeInlineLogFileName(fileIdA, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime1 : commitTime2, 0, TEST_WRITE_TOKEN))
.createNewFile();
new File(fullPartitionPath + FSUtils.makeBaseFileName(commitTime3, TEST_WRITE_TOKEN, fileIdA, BASE_FILE_EXTENSION)).createNewFile();
@@ -1540,7 +1540,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
new File(fullPartitionPath + FSUtils.makeBaseFileName(commitTime2, TEST_WRITE_TOKEN, fileIdB, BASE_FILE_EXTENSION)).createNewFile();
new File(fullPartitionPath + FSUtils.makeBaseFileName(commitTime3, TEST_WRITE_TOKEN, fileIdB, BASE_FILE_EXTENSION)).createNewFile();
new File(fullPartitionPath
- + FSUtils.makeLogFileName(fileIdB, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime3 : commitTime4, 0, TEST_WRITE_TOKEN))
+ + FSUtils.makeInlineLogFileName(fileIdB, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime3 : commitTime4, 0, TEST_WRITE_TOKEN))
.createNewFile();
// file group C
@@ -1662,12 +1662,12 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
new File(fullPartitionPath + "/" + FSUtils.makeBaseFileName(commitTime1, TEST_WRITE_TOKEN, fileIdA, BASE_FILE_EXTENSION))
.createNewFile();
new File(fullPartitionPath + "/"
- + FSUtils.makeLogFileName(fileIdA, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime1 : commitTime2, 0, TEST_WRITE_TOKEN))
+ + FSUtils.makeInlineLogFileName(fileIdA, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime1 : commitTime2, 0, TEST_WRITE_TOKEN))
.createNewFile();
new File(fullPartitionPath + "/" + FSUtils.makeBaseFileName(commitTime3, TEST_WRITE_TOKEN, fileIdA, BASE_FILE_EXTENSION))
.createNewFile();
new File(fullPartitionPath + "/"
- + FSUtils.makeLogFileName(fileIdA, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime3 : commitTime4, 0,
+ + FSUtils.makeInlineLogFileName(fileIdA, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime3 : commitTime4, 0,
TEST_WRITE_TOKEN))
.createNewFile();
@@ -1677,7 +1677,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
new File(fullPartitionPath + "/" + FSUtils.makeBaseFileName(commitTime2, TEST_WRITE_TOKEN, fileIdB, BASE_FILE_EXTENSION))
.createNewFile();
new File(fullPartitionPath + "/"
- + FSUtils.makeLogFileName(fileIdB, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime2 : commitTime3, 0, TEST_WRITE_TOKEN))
+ + FSUtils.makeInlineLogFileName(fileIdB, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? commitTime2 : commitTime3, 0, TEST_WRITE_TOKEN))
.createNewFile();
new File(fullPartitionPath + "/" + FSUtils.makeBaseFileName(commitTime4, TEST_WRITE_TOKEN, fileIdB, BASE_FILE_EXTENSION))
.createNewFile();
@@ -1754,7 +1754,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
String fileId = "fg-X";
String baseFileName = FSUtils.makeBaseFileName(instantTime1, TEST_WRITE_TOKEN, fileId, BASE_FILE_EXTENSION);
- String logFileName1 = FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? instantTime1 : deltaInstantTime3, 0,
+ String logFileName1 = FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? instantTime1 : deltaInstantTime3, 0,
TEST_WRITE_TOKEN);
new File(fullPartitionPath1 + baseFileName).createNewFile();
new File(fullPartitionPath1 + logFileName1).createNewFile();
@@ -1828,9 +1828,9 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
String deltaInstantTime5 = "5";
String deltaInstantTime7 = "7";
String fileName3 =
- FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? compactionRequestedTime : deltaInstantTime5, 0, TEST_WRITE_TOKEN);
+ FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? compactionRequestedTime : deltaInstantTime5, 0, TEST_WRITE_TOKEN);
String fileName4 =
- FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? compactionRequestedTime : deltaInstantTime7, 1, TEST_WRITE_TOKEN);
+ FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, preTableVersion8 ? compactionRequestedTime : deltaInstantTime7, 1, TEST_WRITE_TOKEN);
new File(basePath + "/" + partitionPath1 + "/" + fileName3).createNewFile();
new File(basePath + "/" + partitionPath1 + "/" + fileName4).createNewFile();
new File(basePath + "/" + partitionPath2 + "/" + fileName3).createNewFile();
@@ -2407,8 +2407,8 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
// First delta commit on partitionPath which creates 2 log files.
String commitTime1 = "001";
- String logFileName1 = FSUtils.makeLogFileName(fileId1, HoodieFileFormat.HOODIE_LOG.getFileExtension(), commitTime1, 1, TEST_WRITE_TOKEN);
- String logFileName2 = FSUtils.makeLogFileName(fileId2, HoodieFileFormat.HOODIE_LOG.getFileExtension(), commitTime1, 1, TEST_WRITE_TOKEN);
+ String logFileName1 = FSUtils.makeInlineLogFileName(fileId1, HoodieFileFormat.HOODIE_LOG.getFileExtension(), commitTime1, 1, TEST_WRITE_TOKEN);
+ String logFileName2 = FSUtils.makeInlineLogFileName(fileId2, HoodieFileFormat.HOODIE_LOG.getFileExtension(), commitTime1, 1, TEST_WRITE_TOKEN);
new File(basePath + "/" + partitionPath + "/" + logFileName1).createNewFile();
new File(basePath + "/" + partitionPath + "/" + logFileName2).createNewFile();
expectedState.logFilesCurrentlyPresent.add(logFileName1);
@@ -2432,7 +2432,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
// Second ingestion commit on partitionPath1
// First delta commit on partitionPath1 which creates 2 log files.
String commitTime2 = "002";
- String logFileName3 = FSUtils.makeLogFileName(fileId1, HoodieFileFormat.HOODIE_LOG.getFileExtension(), commitTime2, 2, TEST_WRITE_TOKEN);
+ String logFileName3 = FSUtils.makeInlineLogFileName(fileId1, HoodieFileFormat.HOODIE_LOG.getFileExtension(), commitTime2, 2, TEST_WRITE_TOKEN);
new File(basePath + "/" + partitionPath + "/" + logFileName3).createNewFile();
expectedState.logFilesCurrentlyPresent.add(logFileName3);
@@ -2558,7 +2558,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
// Add log files at instant2
HoodieInstant instant2 = INSTANT_GENERATOR.createNewInstant(State.INFLIGHT, HoodieTimeline.DELTA_COMMIT_ACTION, instantTime2);
- String logFileName2 = FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, instantTime2, 1, TEST_WRITE_TOKEN);
+ String logFileName2 = FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, instantTime2, 1, TEST_WRITE_TOKEN);
new File(basePath + "/" + partitionPath + "/" + logFileName2).createNewFile();
saveAsComplete(commitTimeline, instant2, new HoodieCommitMetadata());
@@ -2597,18 +2597,18 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
// Add log files at instant4
HoodieInstant instant4 = INSTANT_GENERATOR.createNewInstant(State.INFLIGHT, HoodieTimeline.DELTA_COMMIT_ACTION, instantTime4);
- String logFileName4 = FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, instantTime4, 1, TEST_WRITE_TOKEN);
+ String logFileName4 = FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, instantTime4, 1, TEST_WRITE_TOKEN);
new File(basePath + "/" + partitionPath + "/" + logFileName4).createNewFile();
saveAsComplete(commitTimeline, instant4, new HoodieCommitMetadata());
// Add log files at instant5
- String logFileName5 = FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, instantTime5, 1, TEST_WRITE_TOKEN);
+ String logFileName5 = FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, instantTime5, 1, TEST_WRITE_TOKEN);
new File(basePath + "/" + partitionPath + "/" + logFileName5).createNewFile();
commitTimeline.createNewInstant(INSTANT_GENERATOR.createNewInstant(State.REQUESTED, HoodieTimeline.DELTA_COMMIT_ACTION, instantTime5));
commitTimeline.createNewInstant(INSTANT_GENERATOR.createNewInstant(State.INFLIGHT, HoodieTimeline.DELTA_COMMIT_ACTION, instantTime5));
// Add log files at instant6
- String logFileName6 = FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, instantTime5, 1, TEST_WRITE_TOKEN);
+ String logFileName6 = FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, instantTime5, 1, TEST_WRITE_TOKEN);
new File(basePath + "/" + partitionPath + "/" + logFileName6).createNewFile();
commitTimeline.createNewInstant(INSTANT_GENERATOR.createNewInstant(State.REQUESTED, HoodieTimeline.DELTA_COMMIT_ACTION, instantTime6));
commitTimeline.createNewInstant(INSTANT_GENERATOR.createNewInstant(State.INFLIGHT, HoodieTimeline.DELTA_COMMIT_ACTION, instantTime6));
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
index 0e6285f274e0..621c2f23ce1b 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
@@ -987,7 +987,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
return PARTITIONS.stream().flatMap(p -> fileIds.stream().map(f -> {
try {
java.nio.file.Path filePath = Paths.get(basePath, p, deltaCommit
- ? FSUtils.makeLogFileName(f, ".log", instant, 0, TEST_WRITE_TOKEN)
+ ? FSUtils.makeInlineLogFileName(f, ".log", instant, 0, TEST_WRITE_TOKEN)
: FSUtils.makeBaseFileName(instant, TEST_WRITE_TOKEN, f, BASE_FILE_EXTENSION));
Files.createFile(filePath);
HoodieWriteStat w = new HoodieWriteStat();
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
index b2b37ea47fea..f731a3ff0bc0 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
@@ -755,7 +755,7 @@ public class HoodieTestTable implements AutoCloseable {
}
public HoodieTestTable withLogMarkerFile(String partitionPath, String fileId, IOType ioType) throws IOException {
- String logFileName = FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, currentInstantTime, HoodieLogFile.LOGFILE_BASE_VERSION, HoodieLogFormat.UNKNOWN_WRITE_TOKEN);
+ String logFileName = FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, currentInstantTime, HoodieLogFile.LOGFILE_BASE_VERSION, HoodieLogFormat.UNKNOWN_WRITE_TOKEN);
String markerFileName = FileCreateUtils.markerFileName(logFileName, ioType);
FileCreateUtils.createMarkerFile(metaClient, partitionPath, currentInstantTime, markerFileName);
return this;
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
index ac7dac4eb429..3e74ec204b76 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
@@ -126,11 +126,11 @@ public class TestCompactionUtils extends HoodieCommonTestHarness {
// File Slice with no data-file but log files present
FileSlice noDataFileSlice = new FileSlice(DEFAULT_PARTITION_PATHS[0], "000", "noData1");
noDataFileSlice.addLogFile(
- new HoodieLogFile(new StoragePath(FSUtils.makeLogFileName("noData1", ".log", "000", 1,
+ new HoodieLogFile(new StoragePath(FSUtils.makeInlineLogFileName("noData1", ".log", "000", 1,
TEST_WRITE_TOKEN))));
noDataFileSlice.addLogFile(
new HoodieLogFile(new StoragePath(
- FSUtils.makeLogFileName("noData1", ".log", "000", 2, TEST_WRITE_TOKEN))));
+ FSUtils.makeInlineLogFileName("noData1", ".log", "000", 2, TEST_WRITE_TOKEN))));
op = CompactionUtils.buildFromFileSlice(DEFAULT_PARTITION_PATHS[0], noDataFileSlice,
Option.of(metricsCaptureFn));
testFileSliceCompactionOpEquality(noDataFileSlice, op, DEFAULT_PARTITION_PATHS[0],
@@ -141,10 +141,10 @@ public class TestCompactionUtils extends HoodieCommonTestHarness {
fileSlice.setBaseFile(new DummyHoodieBaseFile("/tmp/noLog_1_000" + extension));
fileSlice.addLogFile(
new HoodieLogFile(new StoragePath(
- FSUtils.makeLogFileName("noData1", ".log", "000", 1, TEST_WRITE_TOKEN))));
+ FSUtils.makeInlineLogFileName("noData1", ".log", "000", 1, TEST_WRITE_TOKEN))));
fileSlice.addLogFile(
new HoodieLogFile(new StoragePath(
- FSUtils.makeLogFileName("noData1", ".log", "000", 2, TEST_WRITE_TOKEN))));
+ FSUtils.makeInlineLogFileName("noData1", ".log", "000", 2, TEST_WRITE_TOKEN))));
op = CompactionUtils.buildFromFileSlice(DEFAULT_PARTITION_PATHS[0], fileSlice,
Option.of(metricsCaptureFn));
testFileSliceCompactionOpEquality(fileSlice, op, DEFAULT_PARTITION_PATHS[0],
@@ -165,20 +165,20 @@ public class TestCompactionUtils extends HoodieCommonTestHarness {
new DummyHoodieBaseFile(fullPartitionPath.toString() + "/data1_1_000" + extension));
fileSlice.addLogFile(new HoodieLogFile(
new StoragePath(fullPartitionPath,
- FSUtils.makeLogFileName("noData1", ".log", "000", 1, TEST_WRITE_TOKEN))));
+ FSUtils.makeInlineLogFileName("noData1", ".log", "000", 1, TEST_WRITE_TOKEN))));
fileSlice.addLogFile(new HoodieLogFile(
new StoragePath(fullPartitionPath,
- FSUtils.makeLogFileName("noData1", ".log", "000", 2, TEST_WRITE_TOKEN))));
+ FSUtils.makeInlineLogFileName("noData1", ".log", "000", 2, TEST_WRITE_TOKEN))));
FileSlice noLogFileSlice = new FileSlice(DEFAULT_PARTITION_PATHS[0], "000", "noLog1");
noLogFileSlice.setBaseFile(
new DummyHoodieBaseFile(fullPartitionPath.toString() + "/noLog_1_000" + extension));
FileSlice noDataFileSlice = new FileSlice(DEFAULT_PARTITION_PATHS[0], "000", "noData1");
noDataFileSlice.addLogFile(new HoodieLogFile(
new StoragePath(fullPartitionPath,
- FSUtils.makeLogFileName("noData1", ".log", "000", 1, TEST_WRITE_TOKEN))));
+ FSUtils.makeInlineLogFileName("noData1", ".log", "000", 1, TEST_WRITE_TOKEN))));
noDataFileSlice.addLogFile(new HoodieLogFile(
new StoragePath(fullPartitionPath,
- FSUtils.makeLogFileName("noData1", ".log", "000", 2, TEST_WRITE_TOKEN))));
+ FSUtils.makeInlineLogFileName("noData1", ".log", "000", 2, TEST_WRITE_TOKEN))));
List<FileSlice> fileSliceList = Arrays.asList(emptyFileSlice, noDataFileSlice, fileSlice, noLogFileSlice);
List<Pair<String, FileSlice>> input =
fileSliceList.stream().map(f -> Pair.of(DEFAULT_PARTITION_PATHS[0], f)).collect(Collectors.toList());
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala
index 04df26a839ef..44630e8d5bbc 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala
@@ -551,7 +551,7 @@ class CDCFileGroupIterator(split: HoodieCDCFileGroupSplit,
}
private def isNativeCdcFileSplit(fileSplit: HoodieCDCFileSplit): Boolean = {
- val nativeFlags = fileSplit.getCdcFiles.asScala.map(path => FSUtils.matchNativeLogFile(path).isPresent)
+ val nativeFlags = fileSplit.getCdcFiles.asScala.map(path => FSUtils.isNativeLogFile(path))
ValidationUtils.checkState(nativeFlags.forall(_ == nativeFlags.head),
"CDC file split cannot mix inline and native CDC log files")
nativeFlags.head
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableInsertUpdateDelete.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableInsertUpdateDelete.java
index fcfbb2eb9832..8f2ba3a4947f 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableInsertUpdateDelete.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableInsertUpdateDelete.java
@@ -471,7 +471,8 @@ public class TestHoodieSparkMergeOnReadTableInsertUpdateDelete extends SparkClie
Random random = new Random();
String fakeToken = "";
do {
- fakeToken = Math.abs(random.nextLong()) + "-" + Math.abs(random.nextLong()) + "-" + Math.abs(random.nextLong());
+ fakeToken = random.nextInt(Integer.MAX_VALUE) + "-" + random.nextInt(Integer.MAX_VALUE) + "-"
+ + random.nextInt(Integer.MAX_VALUE);
} while (fakeToken.equals(correctWriteToken));
return fakeToken;
}
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
index e98a832c18b2..f16b0f49fb12 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
@@ -853,7 +853,7 @@ public class TestHoodieMetadataTableValidator extends HoodieSparkClientTestBase
// Empty log file
HoodieLogFile logFile = new HoodieLogFile(new StoragePath(
tempDir.toString(),
- FSUtils.makeLogFileName(
+ FSUtils.makeInlineLogFileName(
UUID.randomUUID().toString(), HoodieLogFile.DELTA_EXTENSION, logInstantTime, 1, "1-0-1")));
storage.create(logFile.getPath()).close();
prepareTimelineAndValidate(metaClient, validator, Collections.emptyList(),
@@ -1285,7 +1285,7 @@ public class TestHoodieMetadataTableValidator extends HoodieSparkClientTestBase
}
private HoodieLogFile generateRandomLogFile(String fileId, String instantTime) {
- return new HoodieLogFile("/dummy/base/" + FSUtils.makeLogFileName(
+ return new HoodieLogFile("/dummy/base/" + FSUtils.makeInlineLogFileName(
fileId, HoodieLogFile.DELTA_EXTENSION, instantTime, 1, "1-0-1"));
}
@@ -1639,8 +1639,8 @@ public class TestHoodieMetadataTableValidator extends HoodieSparkClientTestBase
HoodieBaseFile baseFile = new HoodieBaseFile(FSUtils.makeBaseFileName(
baseInstantTime, "1-0-1", fileId, HoodieFileFormat.PARQUET.getFileExtension()));
List<HoodieLogFile> logFiles = Arrays.asList(
- new HoodieLogFile(FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, baseInstantTime, 1, "1-0-1")),
- new HoodieLogFile(FSUtils.makeLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, baseInstantTime, 2, "1-0-1"))
+ new HoodieLogFile(FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, baseInstantTime, 1, "1-0-1")),
+ new HoodieLogFile(FSUtils.makeInlineLogFileName(fileId, HoodieLogFile.DELTA_EXTENSION, baseInstantTime, 2, "1-0-1"))
);
FileSlice slice = new FileSlice(new HoodieFileGroupId(partition, fileId), baseInstantTime);