You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/05/15 08:02:09 UTC
(hudi) branch branch-0.x updated (2137b2928fc -> e358279a860)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 2137b2928fc [HUDI-6497] Replace FileSystem, Path, and FileStatus usage in hudi-common module (#10591)
new 50f367dbe1f [HUDI-7640] Uses UUID as temporary file suffix for HoodieStorage.createImmutableFileInPath (#11052)
new 5582cde47b7 [HUDI-7618] Add ability to ignore checkpoints in delta streamer (#11018)
new 5020b2a9e18 [HUDI-7643] Fix test by using the right StreamSync constructor (#11056)
new deb90df6409 [HUDI-7515] Fix partition metadata write failure (#10886)
new 82f7485e4f9 [MINOR] Added configurations of Hudi table, file-based SQL source, Hudi error table, and timestamp key generator to configuration listing (#11057)
new 541b4dcf244 [HUDI-7628] Rename FSUtils.getPartitionPath to constructAbsolutePath (#11054)
new 4180f7827fb [HUDI-7631] Clean up usage of CachingPath outside hudi-common module (#11059)
new 273c7332648 [HUDI-7623] Refactoring of RemoteHoodieTableFileSystemView and RequestHandler (#11032)
new beb20386e03 [HUDI-7655] Minor fix to rli validation with MDT validator (#11060)
new 7cadc96b049 [MINOR] Reuse MetadataPartitionType enum to get all partition paths (#11068)
new 278f57828fd [HUDI-7608] Fix Flink table creation configuration not taking effect when writing to Spark (#11005)
new 9a2a61c64f2 [MINOR] Fix incorrect catch of ClassCastException using HoodieSparkKeyGeneratorFactory (#11062)
new 48890500ef6 [MINOR] Fixe naming of methods in HoodieMetadataConfig (#11076)
new 92425585e59 [HUDI-7647] READ_UTC_TIMEZONE doesn't affect log files for MOR tables (#11066)
new 0f6bc8cbd58 [HUDI-6386] Enable testArchivalWithMultiWriters back as they are passing (#9085)
new e358279a860 [MINOR] Fix LoggerName for JDBCExecutor (#11063)
The 16 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 4 +-
.../apache/hudi/cli/commands/RepairsCommand.java | 8 +-
.../apache/hudi/client/CompactionAdminClient.java | 4 +-
.../apache/hudi/config/HoodieErrorTableConfig.java | 3 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 2 +-
.../index/bucket/ConsistentBucketIndexUtils.java | 8 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 4 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 4 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 4 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 4 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 59 ++--
.../hudi/table/action/compact/HoodieCompactor.java | 2 +-
.../table/action/index/RunIndexActionExecutor.java | 2 +-
.../table/action/rollback/BaseRollbackHelper.java | 4 +-
.../rollback/ListingBasedRollbackStrategy.java | 6 +-
.../rollback/MarkerBasedRollbackStrategy.java | 2 +-
.../marker/TimelineServerBasedWriteMarkers.java | 4 +-
.../org/apache/hudi/table/marker/WriteMarkers.java | 2 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 6 +-
.../testutils/HoodieJavaClientTestHarness.java | 2 +-
.../run/strategy/ExecutionStrategyUtil.java | 17 +-
.../MultipleSparkJobExecutionStrategy.java | 3 +-
.../strategy/SingleSparkJobExecutionStrategy.java | 3 +-
.../hudi/client/utils/SparkPartitionUtils.java | 4 +-
.../apache/hudi/index/SparkHoodieIndexFactory.java | 19 +-
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 32 +-
.../factory/HoodieSparkKeyGeneratorFactory.java | 45 ++-
.../hudi/table/HoodieSparkCopyOnWriteTable.java | 12 +-
.../SparkBootstrapCommitActionExecutor.java | 9 +-
.../commit/BaseSparkCommitActionExecutor.java | 9 +-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 20 +-
.../TestSavepointRestoreMergeOnRead.java | 8 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 15 +-
.../java/org/apache/hudi/table/TestCleaner.java | 4 +-
.../hudi/table/marker/TestWriteMarkersBase.java | 2 +-
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 12 +-
.../testutils/HoodieSparkClientTestHarness.java | 2 +-
.../apache/hudi/common/config/ConfigGroups.java | 4 +
.../hudi/common/config/HoodieMetadataConfig.java | 14 +-
.../common/config/TimestampKeyGeneratorConfig.java | 2 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 36 +-
.../hudi/common/model/CompactionOperation.java | 2 +-
.../hudi/common/model/HoodieCommitMetadata.java | 8 +-
.../hudi/common/model/HoodiePartitionMetadata.java | 80 ++---
.../hudi/common/table/HoodieTableConfig.java | 17 +-
.../hudi/common/table/cdc/HoodieCDCExtractor.java | 4 +-
.../table/timeline/HoodieActiveTimeline.java | 6 +-
.../clean/CleanMetadataV1MigrationHandler.java | 2 +-
.../clean/CleanPlanV2MigrationHandler.java | 2 +-
.../compaction/CompactionV1MigrationHandler.java | 2 +-
.../table/view/AbstractTableFileSystemView.java | 4 +-
.../view/HoodieTablePreCommitFileSystemView.java | 4 +-
.../IncrementalTimelineSyncFileSystemView.java | 2 +-
.../view/RemoteHoodieTableFileSystemView.java | 333 +++++++-----------
.../apache/hudi/metadata/BaseTableMetadata.java | 4 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 2 +-
.../apache/hudi/metadata/HoodieTableMetadata.java | 2 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 2 +-
.../hudi/metadata/MetadataPartitionType.java | 12 +
.../common/model/TestHoodiePartitionMetadata.java | 2 +-
.../common/testutils/HoodieTestDataGenerator.java | 5 +-
.../hudi/common/util/TestTablePathUtils.java | 4 +-
.../java/org/apache/hudi/source/FileIndex.java | 2 +-
.../hudi/source/stats/ColumnStatsIndices.java | 2 +-
.../hudi/table/catalog/HoodieHiveCatalog.java | 5 +
.../table/format/mor/MergeOnReadInputFormat.java | 8 +-
.../apache/hudi/util/AvroToRowDataConverters.java | 42 ++-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 2 +-
.../apache/hudi/table/ITTestHoodieDataSource.java | 31 +-
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 40 +++
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 2 +-
.../org/apache/hudi/storage/HoodieStorage.java | 13 +-
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 2 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 12 +-
.../scala/org/apache/hudi/HoodieFileIndex.scala | 2 +-
.../org/apache/hudi/IncrementalRelation.scala | 2 +-
.../org/apache/hudi/RecordLevelIndexSupport.scala | 2 +-
.../apache/hudi/SparkHoodieTableFileIndex.scala | 2 +-
.../SparkFullBootstrapDataProviderBase.java | 70 ++--
.../RepairAddpartitionmetaProcedure.scala | 4 +-
.../RepairMigratePartitionMetaProcedure.scala | 4 +-
.../procedures/ShowInvalidParquetProcedure.scala | 2 +-
.../TestSparkConsistentBucketClustering.java | 2 +-
.../apache/hudi/keygen/TestCustomKeyGenerator.java | 2 +-
.../TestHoodieSparkKeyGeneratorFactory.java | 7 +-
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 2 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 2 +-
.../TestSparkSqlWithCustomKeyGenerator.scala | 4 +-
.../apache/hudi/sync/adb/HoodieAdbJdbcClient.java | 10 +-
.../org/apache/hudi/hive/ddl/HMSDDLExecutor.java | 4 +-
.../org/apache/hudi/hive/ddl/JDBCExecutor.java | 2 +-
.../hudi/hive/ddl/QueryBasedDDLExecutor.java | 4 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 2 +-
.../apache/hudi/sync/common/HoodieSyncClient.java | 4 +-
.../hudi/timeline/service/RequestHandler.java | 374 +++++++++++----------
.../hudi/utilities/HoodieDataTableUtils.java | 2 +-
.../utilities/HoodieMetadataTableValidator.java | 21 +-
.../hudi/utilities/HoodieSnapshotCopier.java | 4 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 4 +-
.../utilities/config/SqlFileBasedSourceConfig.java | 3 +-
.../hudi/utilities/streamer/HoodieStreamer.java | 7 +
.../utilities/streamer/SparkSampleWritesUtils.java | 15 +-
.../apache/hudi/utilities/streamer/StreamSync.java | 13 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 4 +-
.../streamer/TestStreamSyncUnitTests.java | 61 ++++
105 files changed, 847 insertions(+), 860 deletions(-)
(hudi) 04/16: [HUDI-7515] Fix partition metadata write failure (#10886)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit deb90df6409723c052aa11fc2208934fc2e10519
Author: Wechar Yu <we...@shopee.com>
AuthorDate: Sat Apr 20 08:18:18 2024 +0800
[HUDI-7515] Fix partition metadata write failure (#10886)
---
.../apache/hudi/cli/commands/RepairsCommand.java | 4 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 2 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 2 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 2 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 2 +-
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 2 +-
.../hudi/common/model/HoodiePartitionMetadata.java | 80 ++++++++++++----------
.../table/timeline/HoodieActiveTimeline.java | 6 +-
.../common/model/TestHoodiePartitionMetadata.java | 2 +-
.../common/testutils/HoodieTestDataGenerator.java | 5 +-
.../hudi/common/util/TestTablePathUtils.java | 4 +-
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 2 +-
.../org/apache/hudi/storage/HoodieStorage.java | 2 +-
.../RepairAddpartitionmetaProcedure.scala | 2 +-
.../RepairMigratePartitionMetaProcedure.scala | 2 +-
15 files changed, 60 insertions(+), 59 deletions(-)
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
index a41e57a0bb2..28e1a0d39ba 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
@@ -139,7 +139,7 @@ public class RepairsCommand {
HoodiePartitionMetadata partitionMetadata =
new HoodiePartitionMetadata(HoodieCLI.storage, latestCommit, basePath, partitionPath,
client.getTableConfig().getPartitionMetafileFormat());
- partitionMetadata.trySave(0);
+ partitionMetadata.trySave();
row[2] = "Repaired";
}
}
@@ -256,7 +256,7 @@ public class RepairsCommand {
HoodiePartitionMetadata partitionMetadata =
new HoodiePartitionMetadata(HoodieCLI.storage, latestCommit, basePath, partition,
Option.of(client.getTableConfig().getBaseFileFormat()));
- partitionMetadata.trySave(0);
+ partitionMetadata.trySave();
}
// delete it, in case we failed midway last time.
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
index 40613e15b1f..2bac318fc81 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
@@ -208,7 +208,7 @@ public class HoodieAppendHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O
HoodiePartitionMetadata partitionMetadata = new HoodiePartitionMetadata(storage, baseInstantTime,
new StoragePath(config.getBasePath()), FSUtils.getPartitionPath(config.getBasePath(), partitionPath),
hoodieTable.getPartitionMetafileFormat());
- partitionMetadata.trySave(getPartitionId());
+ partitionMetadata.trySave();
this.writer = createLogWriter(fileSlice, baseInstantTime);
} catch (Exception e) {
LOG.error("Error in update task at commit " + instantTime, e);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
index 6f3824ac34c..0ad4e212a1a 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
@@ -100,7 +100,7 @@ public class HoodieCreateHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O
new StoragePath(config.getBasePath()),
FSUtils.getPartitionPath(config.getBasePath(), partitionPath),
hoodieTable.getPartitionMetafileFormat());
- partitionMetadata.trySave(getPartitionId());
+ partitionMetadata.trySave();
createMarkerFile(partitionPath,
FSUtils.makeBaseFileName(this.instantTime, this.writeToken, this.fileId, hoodieTable.getBaseFileExtension()));
this.fileWriter =
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
index 55aa334a97a..afae82fd13f 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
@@ -176,7 +176,7 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
new StoragePath(config.getBasePath()),
FSUtils.getPartitionPath(config.getBasePath(), partitionPath),
hoodieTable.getPartitionMetafileFormat());
- partitionMetadata.trySave(getPartitionId());
+ partitionMetadata.trySave();
String newFileName = FSUtils.makeBaseFileName(instantTime, writeToken, fileId, hoodieTable.getBaseFileExtension());
makeOldAndNewFilePaths(partitionPath, latestValidFilePath, newFileName);
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
index 56e38dc8ddf..19455773153 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
@@ -106,7 +106,7 @@ public class HoodieRowDataCreateHandle implements Serializable {
new StoragePath(writeConfig.getBasePath()),
FSUtils.getPartitionPath(writeConfig.getBasePath(), partitionPath),
table.getPartitionMetafileFormat());
- partitionMetadata.trySave(taskPartitionId);
+ partitionMetadata.trySave();
createMarkerFile(partitionPath, FSUtils.makeBaseFileName(this.instantTime, getWriteToken(), this.fileId, table.getBaseFileExtension()));
this.fileWriter = createNewFileWriter(path, table, writeConfig, rowType);
} catch (IOException e) {
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
index 98341bf62b4..890b12899f1 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
@@ -143,7 +143,7 @@ public class HoodieRowCreateHandle implements Serializable {
new StoragePath(writeConfig.getBasePath()),
FSUtils.getPartitionPath(writeConfig.getBasePath(), partitionPath),
table.getPartitionMetafileFormat());
- partitionMetadata.trySave(taskPartitionId);
+ partitionMetadata.trySave();
createMarkerFile(partitionPath, fileName, instantTime, table, writeConfig);
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
index adeaaa5be4f..a90d05aefdd 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
@@ -20,6 +20,7 @@ package org.apache.hudi.common.model;
import org.apache.hudi.common.util.BaseFileUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.RetryHelper;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
@@ -30,12 +31,13 @@ import org.apache.hadoop.conf.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -94,36 +96,29 @@ public class HoodiePartitionMetadata {
/**
* Write the metadata safely into partition atomically.
*/
- public void trySave(int taskPartitionId) {
- String extension = getMetafileExtension();
- StoragePath tmpMetaPath =
- new StoragePath(partitionPath, HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE_PREFIX + "_" + taskPartitionId + extension);
- StoragePath metaPath = new StoragePath(partitionPath, HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE_PREFIX + extension);
- boolean metafileExists = false;
+ public void trySave() throws HoodieIOException {
+ StoragePath metaPath = new StoragePath(
+ partitionPath, HOODIE_PARTITION_METAFILE_PREFIX + getMetafileExtension());
- try {
- metafileExists = storage.exists(metaPath);
- if (!metafileExists) {
- // write to temporary file
- writeMetafile(tmpMetaPath);
- // move to actual path
- storage.rename(tmpMetaPath, metaPath);
- }
- } catch (IOException ioe) {
- LOG.warn("Error trying to save partition metadata (this is okay, as long as at least 1 of these succeeded), "
- + partitionPath, ioe);
- } finally {
- if (!metafileExists) {
- try {
- // clean up tmp file, if still lying around
- if (storage.exists(tmpMetaPath)) {
- storage.deleteFile(tmpMetaPath);
+ // This retry mechanism enables an exit-fast in metaPath exists check, which avoid the
+ // tasks failures when there are two or more tasks trying to create the same metaPath.
+ RetryHelper<Void, HoodieIOException> retryHelper = new RetryHelper(1000, 3, 1000, HoodieIOException.class.getName())
+ .tryWith(() -> {
+ if (!storage.exists(metaPath)) {
+ if (format.isPresent()) {
+ writeMetafileInFormat(metaPath, format.get());
+ } else {
+ // Backwards compatible properties file format
+ try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {
+ props.store(os, "partition metadata");
+ Option content = Option.of(os.toByteArray());
+ storage.createImmutableFileInPath(metaPath, content);
+ }
+ }
}
- } catch (IOException ioe) {
- LOG.warn("Error trying to clean up temporary files for " + partitionPath, ioe);
- }
- }
- }
+ return null;
+ });
+ retryHelper.start();
}
private String getMetafileExtension() {
@@ -134,17 +129,26 @@ public class HoodiePartitionMetadata {
/**
* Write the partition metadata in the correct format in the given file path.
*
- * @param filePath path of the file to write.
+ * @param filePath Path of the file to write
+ * @param format Hoodie table file format
* @throws IOException
*/
- private void writeMetafile(StoragePath filePath) throws IOException {
- if (format.isPresent()) {
- BaseFileUtils.getInstance(format.get()).writeMetaFile(storage, filePath, props);
- } else {
- // Backwards compatible properties file format
- try (OutputStream os = storage.create(filePath, true)) {
- props.store(os, "partition metadata");
- os.flush();
+ private void writeMetafileInFormat(StoragePath filePath, HoodieFileFormat format) throws IOException {
+ StoragePath tmpPath = new StoragePath(partitionPath,
+ HOODIE_PARTITION_METAFILE_PREFIX + "_" + UUID.randomUUID() + getMetafileExtension());
+ try {
+ // write to temporary file
+ BaseFileUtils.getInstance(format).writeMetaFile(storage, tmpPath, props);
+ // move to actual path
+ storage.rename(tmpPath, filePath);
+ } finally {
+ try {
+ // clean up tmp file, if still lying around
+ if (storage.exists(tmpPath)) {
+ storage.deleteFile(tmpPath);
+ }
+ } catch (IOException ioe) {
+ LOG.warn("Error trying to clean up temporary files for " + partitionPath, ioe);
}
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
index 0545fe392fc..3c8d6aa4306 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
@@ -796,11 +796,7 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
if (allowOverwrite || metaClient.getTimelineLayoutVersion().isNullVersion()) {
FileIOUtils.createFileInPath(metaClient.getStorage(), fullPath, content);
} else {
- try {
- metaClient.getStorage().createImmutableFileInPath(fullPath, content);
- } catch (IOException e) {
- throw new HoodieIOException("Cannot create immutable file: " + fullPath, e);
- }
+ metaClient.getStorage().createImmutableFileInPath(fullPath, content);
}
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java
index 70474ec833f..ef01aa7deed 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java
@@ -77,7 +77,7 @@ public class TestHoodiePartitionMetadata extends HoodieCommonTestHarness {
HoodiePartitionMetadata writtenMetadata = new HoodiePartitionMetadata(
metaClient.getStorage(), commitTime, new StoragePath(basePath), partitionPath,
format);
- writtenMetadata.trySave(0);
+ writtenMetadata.trySave();
// when
HoodiePartitionMetadata readMetadata = new HoodiePartitionMetadata(
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
index 9cb2ab3bfb7..a7440f8993a 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
@@ -253,7 +253,8 @@ public class HoodieTestDataGenerator implements AutoCloseable {
* @deprecated please use non-static version
*/
public static void writePartitionMetadataDeprecated(HoodieStorage storage,
- String[] partitionPaths, String basePath) {
+ String[] partitionPaths,
+ String basePath) {
new HoodieTestDataGenerator().writePartitionMetadata(storage, partitionPaths, basePath);
}
@@ -268,7 +269,7 @@ public class HoodieTestDataGenerator implements AutoCloseable {
String basePath) {
for (String partitionPath : partitionPaths) {
new HoodiePartitionMetadata(storage, "000", new StoragePath(basePath),
- new StoragePath(basePath, partitionPath), Option.empty()).trySave(0);
+ new StoragePath(basePath, partitionPath), Option.empty()).trySave();
}
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java
index 2022ee8cfda..0db5c207463 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java
@@ -82,11 +82,11 @@ public final class TestTablePathUtils {
HoodiePartitionMetadata partitionMetadata1 = new HoodiePartitionMetadata(
storage, Instant.now().toString(), tablePath,
partitionPath1, partitionMetafileFormat);
- partitionMetadata1.trySave(1);
+ partitionMetadata1.trySave();
HoodiePartitionMetadata partitionMetadata2 = new HoodiePartitionMetadata(
storage, Instant.now().toString(), tablePath,
partitionPath2, partitionMetafileFormat);
- partitionMetadata2.trySave(2);
+ partitionMetadata2.trySave();
// Create files
URI filePathURI1 =
diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java
index f208bd0e3c6..cfdd6c88395 100644
--- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java
+++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java
@@ -512,7 +512,7 @@ public class InputFormatTestUtil {
new StoragePath(partitionPath.toAbsolutePath().toString()),
Option.of(HoodieFileFormat.PARQUET));
- partitionMetadata.trySave((int) (Math.random() * 1000));
+ partitionMetadata.trySave();
}
}
diff --git a/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java b/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
index be160caba3b..b8735cc89d9 100644
--- a/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
+++ b/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
@@ -257,7 +257,7 @@ public abstract class HoodieStorage implements Closeable {
*/
@PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
public final void createImmutableFileInPath(StoragePath path,
- Option<byte[]> content) throws IOException {
+ Option<byte[]> content) throws HoodieIOException {
OutputStream fsout = null;
StoragePath tmpPath = null;
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala
index d13895af414..03ef6cc3f54 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala
@@ -70,7 +70,7 @@ class RepairAddpartitionmetaProcedure extends BaseProcedure with ProcedureBuilde
if (!dryRun) {
val partitionMetadata: HoodiePartitionMetadata = new HoodiePartitionMetadata(
metaClient.getStorage, latestCommit, basePath, partitionPath, metaClient.getTableConfig.getPartitionMetafileFormat)
- partitionMetadata.trySave(0)
+ partitionMetadata.trySave()
action = "Repaired"
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
index 5651055ee99..07fd7c92a68 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
@@ -78,7 +78,7 @@ class RepairMigratePartitionMetaProcedure extends BaseProcedure with ProcedureBu
if (!baseFormatFile.isPresent) {
val partitionMetadata: HoodiePartitionMetadata = new HoodiePartitionMetadata(metaClient.getStorage, latestCommit,
basePath, partition, Option.of(metaClient.getTableConfig.getBaseFileFormat))
- partitionMetadata.trySave(0)
+ partitionMetadata.trySave()
}
// delete it, in case we failed midway last time.
textFormatFile.ifPresent(
(hudi) 08/16: [HUDI-7623] Refactoring of RemoteHoodieTableFileSystemView and RequestHandler (#11032)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 273c7332648c05f06a871dbadbd1d6f6acab917b
Author: Vova Kolmakov <wo...@gmail.com>
AuthorDate: Wed May 15 00:56:54 2024 -0700
[HUDI-7623] Refactoring of RemoteHoodieTableFileSystemView and RequestHandler (#11032)
---
.../view/RemoteHoodieTableFileSystemView.java | 333 +++++++-----------
.../hudi/timeline/service/RequestHandler.java | 374 +++++++++++----------
2 files changed, 315 insertions(+), 392 deletions(-)
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java
index 6c8295fd75f..7de9119992e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java
@@ -36,11 +36,11 @@ import org.apache.hudi.common.table.timeline.dto.InstantDTO;
import org.apache.hudi.common.table.timeline.dto.TimelineDTO;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.RetryHelper;
-import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieRemoteException;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.module.afterburner.AfterburnerModule;
@@ -66,66 +66,46 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().registerModule(new AfterburnerModule());
+ private static final String SCHEME = "http";
private static final String BASE_URL = "/v1/hoodie/view";
public static final String LATEST_PARTITION_SLICES_URL = String.format("%s/%s", BASE_URL, "slices/partition/latest/");
public static final String LATEST_PARTITION_SLICES_STATELESS_URL = String.format("%s/%s", BASE_URL, "slices/partition/latest/stateless/");
public static final String LATEST_PARTITION_SLICE_URL = String.format("%s/%s", BASE_URL, "slices/file/latest/");
- public static final String LATEST_PARTITION_UNCOMPACTED_SLICES_URL =
- String.format("%s/%s", BASE_URL, "slices/uncompacted/partition/latest/");
+ public static final String LATEST_PARTITION_UNCOMPACTED_SLICES_URL = String.format("%s/%s", BASE_URL, "slices/uncompacted/partition/latest/");
public static final String ALL_SLICES_URL = String.format("%s/%s", BASE_URL, "slices/all");
- public static final String LATEST_SLICES_MERGED_BEFORE_ON_INSTANT_URL =
- String.format("%s/%s", BASE_URL, "slices/merged/beforeoron/latest/");
+ public static final String LATEST_SLICES_MERGED_BEFORE_ON_INSTANT_URL = String.format("%s/%s", BASE_URL, "slices/merged/beforeoron/latest/");
public static final String LATEST_SLICES_RANGE_INSTANT_URL = String.format("%s/%s", BASE_URL, "slices/range/latest/");
- public static final String LATEST_SLICES_BEFORE_ON_INSTANT_URL =
- String.format("%s/%s", BASE_URL, "slices/beforeoron/latest/");
- public static final String ALL_LATEST_SLICES_BEFORE_ON_INSTANT_URL =
- String.format("%s/%s", BASE_URL, "slices/all/beforeoron/latest/");
-
- public static final String PENDING_COMPACTION_OPS = String.format("%s/%s", BASE_URL, "compactions/pending/");
- public static final String PENDING_LOG_COMPACTION_OPS = String.format("%s/%s", BASE_URL, "logcompactions/pending/");
-
- public static final String LATEST_PARTITION_DATA_FILES_URL =
- String.format("%s/%s", BASE_URL, "datafiles/latest/partition");
- public static final String LATEST_PARTITION_DATA_FILE_URL =
- String.format("%s/%s", BASE_URL, "datafile/latest/partition");
- public static final String ALL_DATA_FILES = String.format("%s/%s", BASE_URL, "datafiles/all");
- public static final String LATEST_ALL_DATA_FILES = String.format("%s/%s", BASE_URL, "datafiles/all/latest/");
- public static final String LATEST_DATA_FILE_ON_INSTANT_URL = String.format("%s/%s", BASE_URL, "datafile/on/latest/");
-
- public static final String LATEST_DATA_FILES_RANGE_INSTANT_URL =
- String.format("%s/%s", BASE_URL, "datafiles/range/latest/");
- public static final String LATEST_DATA_FILES_BEFORE_ON_INSTANT_URL =
- String.format("%s/%s", BASE_URL, "datafiles/beforeoron/latest/");
- public static final String ALL_LATEST_BASE_FILES_BEFORE_ON_INSTANT_URL =
- String.format("%s/%s", BASE_URL, "basefiles/all/beforeoron/");
-
- public static final String ALL_FILEGROUPS_FOR_PARTITION_URL =
- String.format("%s/%s", BASE_URL, "filegroups/all/partition/");
-
- public static final String ALL_FILEGROUPS_FOR_PARTITION_STATELESS_URL =
- String.format("%s/%s", BASE_URL, "filegroups/all/partition/stateless/");
-
- public static final String ALL_REPLACED_FILEGROUPS_BEFORE_OR_ON =
- String.format("%s/%s", BASE_URL, "filegroups/replaced/beforeoron/");
-
- public static final String ALL_REPLACED_FILEGROUPS_BEFORE =
- String.format("%s/%s", BASE_URL, "filegroups/replaced/before/");
+ public static final String LATEST_SLICES_BEFORE_ON_INSTANT_URL = String.format("%s/%s", BASE_URL, "slices/beforeoron/latest/");
+ public static final String ALL_LATEST_SLICES_BEFORE_ON_INSTANT_URL = String.format("%s/%s", BASE_URL, "slices/all/beforeoron/latest/");
- public static final String ALL_REPLACED_FILEGROUPS_AFTER_OR_ON =
- String.format("%s/%s", BASE_URL, "filegroups/replaced/afteroron/");
+ public static final String PENDING_COMPACTION_OPS_URL = String.format("%s/%s", BASE_URL, "compactions/pending/");
+ public static final String PENDING_LOG_COMPACTION_OPS_URL = String.format("%s/%s", BASE_URL, "logcompactions/pending/");
- public static final String ALL_REPLACED_FILEGROUPS_PARTITION =
- String.format("%s/%s", BASE_URL, "filegroups/replaced/partition/");
+ public static final String LATEST_PARTITION_DATA_FILES_URL = String.format("%s/%s", BASE_URL, "datafiles/latest/partition");
+ public static final String LATEST_PARTITION_DATA_FILE_URL = String.format("%s/%s", BASE_URL, "datafile/latest/partition");
+ public static final String ALL_DATA_FILES_URL = String.format("%s/%s", BASE_URL, "datafiles/all");
+ public static final String LATEST_ALL_DATA_FILES_URL = String.format("%s/%s", BASE_URL, "datafiles/all/latest/");
+ public static final String LATEST_DATA_FILE_ON_INSTANT_URL = String.format("%s/%s", BASE_URL, "datafile/on/latest/");
+ public static final String LATEST_DATA_FILES_RANGE_INSTANT_URL = String.format("%s/%s", BASE_URL, "datafiles/range/latest/");
+ public static final String LATEST_DATA_FILES_BEFORE_ON_INSTANT_URL = String.format("%s/%s", BASE_URL, "datafiles/beforeoron/latest/");
+ public static final String ALL_LATEST_BASE_FILES_BEFORE_ON_INSTANT_URL = String.format("%s/%s", BASE_URL, "basefiles/all/beforeoron/");
+
+ public static final String ALL_FILEGROUPS_FOR_PARTITION_URL = String.format("%s/%s", BASE_URL, "filegroups/all/partition/");
+ public static final String ALL_FILEGROUPS_FOR_PARTITION_STATELESS_URL = String.format("%s/%s", BASE_URL, "filegroups/all/partition/stateless/");
+ public static final String ALL_REPLACED_FILEGROUPS_BEFORE_OR_ON_URL = String.format("%s/%s", BASE_URL, "filegroups/replaced/beforeoron/");
+ public static final String ALL_REPLACED_FILEGROUPS_BEFORE_URL = String.format("%s/%s", BASE_URL, "filegroups/replaced/before/");
+ public static final String ALL_REPLACED_FILEGROUPS_AFTER_OR_ON_URL = String.format("%s/%s", BASE_URL, "filegroups/replaced/afteroron/");
+ public static final String ALL_REPLACED_FILEGROUPS_PARTITION_URL = String.format("%s/%s", BASE_URL, "filegroups/replaced/partition/");
- public static final String PENDING_CLUSTERING_FILEGROUPS = String.format("%s/%s", BASE_URL, "clustering/pending/");
+ public static final String PENDING_CLUSTERING_FILEGROUPS_URL = String.format("%s/%s", BASE_URL, "clustering/pending/");
- public static final String LAST_INSTANT = String.format("%s/%s", BASE_URL, "timeline/instant/last");
- public static final String LAST_INSTANTS = String.format("%s/%s", BASE_URL, "timeline/instants/last");
+ public static final String LAST_INSTANT_URL = String.format("%s/%s", BASE_URL, "timeline/instant/last");
+ public static final String LAST_INSTANTS_URL = String.format("%s/%s", BASE_URL, "timeline/instants/last");
- public static final String TIMELINE = String.format("%s/%s", BASE_URL, "timeline/instants/all");
+ public static final String TIMELINE_URL = String.format("%s/%s", BASE_URL, "timeline/instants/all");
// POST Requests
- public static final String REFRESH_TABLE = String.format("%s/%s", BASE_URL, "refresh/");
+ public static final String REFRESH_TABLE_URL = String.format("%s/%s", BASE_URL, "refresh/");
public static final String LOAD_ALL_PARTITIONS_URL = String.format("%s/%s", BASE_URL, "loadallpartitions/");
public static final String LOAD_PARTITIONS_URL = String.format("%s/%s", BASE_URL, "loadpartitions/");
@@ -142,6 +122,7 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
public static final String REFRESH_OFF = "refreshoff";
public static final String INCLUDE_FILES_IN_PENDING_COMPACTION_PARAM = "includependingcompaction";
+ public static final String MULTI_VALUE_SEPARATOR = ",";
private static final Logger LOG = LoggerFactory.getLogger(RemoteHoodieTableFileSystemView.class);
private static final TypeReference<List<FileSliceDTO>> FILE_SLICE_DTOS_REFERENCE = new TypeReference<List<FileSliceDTO>>() {};
@@ -175,7 +156,7 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
}
public RemoteHoodieTableFileSystemView(HoodieTableMetaClient metaClient, FileSystemViewStorageConfig viewConf) {
- this.basePath = metaClient.getBasePath();
+ this.basePath = metaClient.getBasePathV2().toString();
this.metaClient = metaClient;
this.timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
this.serverHost = viewConf.getRemoteViewServerHost();
@@ -195,9 +176,7 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
RequestMethod method) throws IOException {
ValidationUtils.checkArgument(!closed, "View already closed");
- URIBuilder builder =
- new URIBuilder().setHost(serverHost).setPort(serverPort).setPath(requestPath).setScheme("http");
-
+ URIBuilder builder = new URIBuilder().setHost(serverHost).setPort(serverPort).setPath(requestPath).setScheme(SCHEME);
queryParameters.forEach(builder::addParameter);
// Adding mandatory parameters - Last instants affecting file-slice
@@ -205,7 +184,7 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
builder.addParameter(TIMELINE_HASH, timeline.getTimelineHash());
String url = builder.toString();
- LOG.info("Sending request : (" + url + ")");
+ LOG.info("Sending request : ({})", url);
Response response = retryHelper != null ? retryHelper.start(() -> get(timeoutMs, url, method)) : get(timeoutMs, url, method);
String content = response.returnContent().asString(Consts.UTF_8);
return (T) OBJECT_MAPPER.readValue(content, reference);
@@ -251,32 +230,32 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
return paramsMap;
}
+ private Stream<HoodieBaseFile> getLatestBaseFilesFromParams(String requestPath, Map<String, String> paramsMap) {
+ try {
+ List<BaseFileDTO> dataFiles = executeRequest(requestPath, paramsMap,
+ BASE_FILE_DTOS_REFERENCE, RequestMethod.GET);
+ return dataFiles.stream().map(BaseFileDTO::toHoodieBaseFile);
+ } catch (IOException e) {
+ throw new HoodieRemoteException(e);
+ }
+ }
+
@Override
public Stream<HoodieBaseFile> getLatestBaseFiles(String partitionPath) {
Map<String, String> paramsMap = getParamsWithPartitionPath(partitionPath);
- return getLatestBaseFilesFromParams(paramsMap, LATEST_PARTITION_DATA_FILES_URL);
+ return getLatestBaseFilesFromParams(LATEST_PARTITION_DATA_FILES_URL, paramsMap);
}
@Override
public Stream<HoodieBaseFile> getLatestBaseFiles() {
Map<String, String> paramsMap = getParams();
- return getLatestBaseFilesFromParams(paramsMap, LATEST_ALL_DATA_FILES);
- }
-
- private Stream<HoodieBaseFile> getLatestBaseFilesFromParams(Map<String, String> paramsMap, String requestPath) {
- try {
- List<BaseFileDTO> dataFiles = executeRequest(requestPath, paramsMap,
- BASE_FILE_DTOS_REFERENCE, RequestMethod.GET);
- return dataFiles.stream().map(BaseFileDTO::toHoodieBaseFile);
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ return getLatestBaseFilesFromParams(LATEST_ALL_DATA_FILES_URL, paramsMap);
}
@Override
public Stream<HoodieBaseFile> getLatestBaseFilesBeforeOrOn(String partitionPath, String maxCommitTime) {
Map<String, String> paramsMap = getParamsWithAdditionalParam(partitionPath, MAX_INSTANT_PARAM, maxCommitTime);
- return getLatestBaseFilesFromParams(paramsMap, LATEST_DATA_FILES_BEFORE_ON_INSTANT_URL);
+ return getLatestBaseFilesFromParams(LATEST_DATA_FILES_BEFORE_ON_INSTANT_URL, paramsMap);
}
@Override
@@ -304,35 +283,30 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
public Option<HoodieBaseFile> getBaseFileOn(String partitionPath, String instantTime, String fileId) {
Map<String, String> paramsMap = getParamsWithAdditionalParams(partitionPath,
new String[] {INSTANT_PARAM, FILEID_PARAM}, new String[] {instantTime, fileId});
- try {
- List<BaseFileDTO> dataFiles = executeRequest(LATEST_DATA_FILE_ON_INSTANT_URL, paramsMap,
- BASE_FILE_DTOS_REFERENCE, RequestMethod.GET);
- return Option.fromJavaOptional(dataFiles.stream()
- .map(BaseFileDTO::toHoodieBaseFile)
- .findFirst());
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ return Option.fromJavaOptional(getLatestBaseFilesFromParams(LATEST_DATA_FILE_ON_INSTANT_URL, paramsMap).findFirst());
}
@Override
public Stream<HoodieBaseFile> getLatestBaseFilesInRange(List<String> commitsToReturn) {
- Map<String, String> paramsMap =
- getParams(INSTANTS_PARAM, StringUtils.join(commitsToReturn.toArray(new String[0]), ","));
- return getLatestBaseFilesFromParams(paramsMap, LATEST_DATA_FILES_RANGE_INSTANT_URL);
+ Map<String, String> paramsMap = getParams(INSTANTS_PARAM, String.join(MULTI_VALUE_SEPARATOR, commitsToReturn));
+ return getLatestBaseFilesFromParams(LATEST_DATA_FILES_RANGE_INSTANT_URL, paramsMap);
}
@Override
public Stream<HoodieBaseFile> getAllBaseFiles(String partitionPath) {
Map<String, String> paramsMap = getParamsWithPartitionPath(partitionPath);
- return getLatestBaseFilesFromParams(paramsMap, ALL_DATA_FILES);
+ return getLatestBaseFilesFromParams(ALL_DATA_FILES_URL, paramsMap);
}
@Override
- public Stream<FileSlice> getLatestFileSlices(String partitionPath) {
- Map<String, String> paramsMap = getParamsWithPartitionPath(partitionPath);
+ public Option<HoodieBaseFile> getLatestBaseFile(String partitionPath, String fileId) {
+ Map<String, String> paramsMap = getParamsWithAdditionalParam(partitionPath, FILEID_PARAM, fileId);
+ return Option.fromJavaOptional(getLatestBaseFilesFromParams(LATEST_PARTITION_DATA_FILE_URL, paramsMap).findFirst());
+ }
+
+ private Stream<FileSlice> getLatestFileSlicesStreamFromParams(String requestPath, Map<String, String> paramsMap) {
try {
- List<FileSliceDTO> dataFiles = executeRequest(LATEST_PARTITION_SLICES_URL, paramsMap,
+ List<FileSliceDTO> dataFiles = executeRequest(requestPath, paramsMap,
FILE_SLICE_DTOS_REFERENCE, RequestMethod.GET);
return dataFiles.stream().map(FileSliceDTO::toFileSlice);
} catch (IOException e) {
@@ -340,40 +314,28 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
}
}
+ @Override
+ public Stream<FileSlice> getLatestFileSlices(String partitionPath) {
+ Map<String, String> paramsMap = getParamsWithPartitionPath(partitionPath);
+ return getLatestFileSlicesStreamFromParams(LATEST_PARTITION_SLICES_URL, paramsMap);
+ }
+
@Override
public Stream<FileSlice> getLatestFileSlicesStateless(String partitionPath) {
Map<String, String> paramsMap = getParamsWithPartitionPath(partitionPath);
- try {
- List<FileSliceDTO> dataFiles = executeRequest(LATEST_PARTITION_SLICES_STATELESS_URL, paramsMap,
- new TypeReference<List<FileSliceDTO>>() {}, RequestMethod.GET);
- return dataFiles.stream().map(FileSliceDTO::toFileSlice);
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ return getLatestFileSlicesStreamFromParams(LATEST_PARTITION_SLICES_STATELESS_URL, paramsMap);
}
@Override
public Option<FileSlice> getLatestFileSlice(String partitionPath, String fileId) {
Map<String, String> paramsMap = getParamsWithAdditionalParam(partitionPath, FILEID_PARAM, fileId);
- try {
- List<FileSliceDTO> dataFiles = executeRequest(LATEST_PARTITION_SLICE_URL, paramsMap,
- FILE_SLICE_DTOS_REFERENCE, RequestMethod.GET);
- return Option.fromJavaOptional(dataFiles.stream().map(FileSliceDTO::toFileSlice).findFirst());
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ return Option.fromJavaOptional(getLatestFileSlicesStreamFromParams(LATEST_PARTITION_SLICE_URL, paramsMap).findFirst());
}
@Override
public Stream<FileSlice> getLatestUnCompactedFileSlices(String partitionPath) {
Map<String, String> paramsMap = getParamsWithPartitionPath(partitionPath);
- try {
- List<FileSliceDTO> dataFiles = executeRequest(LATEST_PARTITION_UNCOMPACTED_SLICES_URL, paramsMap,
- FILE_SLICE_DTOS_REFERENCE, RequestMethod.GET);
- return dataFiles.stream().map(FileSliceDTO::toFileSlice);
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ return getLatestFileSlicesStreamFromParams(LATEST_PARTITION_UNCOMPACTED_SLICES_URL, paramsMap);
}
@Override
@@ -382,13 +344,7 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
Map<String, String> paramsMap = getParamsWithAdditionalParams(partitionPath,
new String[] {MAX_INSTANT_PARAM, INCLUDE_FILES_IN_PENDING_COMPACTION_PARAM},
new String[] {maxCommitTime, String.valueOf(includeFileSlicesInPendingCompaction)});
- try {
- List<FileSliceDTO> dataFiles = executeRequest(LATEST_SLICES_BEFORE_ON_INSTANT_URL, paramsMap,
- FILE_SLICE_DTOS_REFERENCE, RequestMethod.GET);
- return dataFiles.stream().map(FileSliceDTO::toFileSlice);
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ return getLatestFileSlicesStreamFromParams(LATEST_SLICES_BEFORE_ON_INSTANT_URL, paramsMap);
}
@Override
@@ -412,35 +368,26 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
@Override
public Stream<FileSlice> getLatestMergedFileSlicesBeforeOrOn(String partitionPath, String maxInstantTime) {
Map<String, String> paramsMap = getParamsWithAdditionalParam(partitionPath, MAX_INSTANT_PARAM, maxInstantTime);
- try {
- List<FileSliceDTO> dataFiles = executeRequest(LATEST_SLICES_MERGED_BEFORE_ON_INSTANT_URL, paramsMap,
- FILE_SLICE_DTOS_REFERENCE, RequestMethod.GET);
- return dataFiles.stream().map(FileSliceDTO::toFileSlice);
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ return getLatestFileSlicesStreamFromParams(LATEST_SLICES_MERGED_BEFORE_ON_INSTANT_URL, paramsMap);
}
@Override
public Stream<FileSlice> getLatestFileSliceInRange(List<String> commitsToReturn) {
- Map<String, String> paramsMap =
- getParams(INSTANTS_PARAM, StringUtils.join(commitsToReturn.toArray(new String[0]), ","));
- try {
- List<FileSliceDTO> dataFiles = executeRequest(LATEST_SLICES_RANGE_INSTANT_URL, paramsMap,
- FILE_SLICE_DTOS_REFERENCE, RequestMethod.GET);
- return dataFiles.stream().map(FileSliceDTO::toFileSlice);
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ Map<String, String> paramsMap = getParams(INSTANTS_PARAM, String.join(MULTI_VALUE_SEPARATOR, commitsToReturn));
+ return getLatestFileSlicesStreamFromParams(LATEST_SLICES_RANGE_INSTANT_URL, paramsMap);
}
@Override
public Stream<FileSlice> getAllFileSlices(String partitionPath) {
Map<String, String> paramsMap = getParamsWithPartitionPath(partitionPath);
+ return getLatestFileSlicesStreamFromParams(ALL_SLICES_URL, paramsMap);
+ }
+
+ private Stream<HoodieFileGroup> getAllFileGroupsForPartitionFromParams(String requestPath, Map<String, String> paramsMap) {
try {
- List<FileSliceDTO> dataFiles =
- executeRequest(ALL_SLICES_URL, paramsMap, FILE_SLICE_DTOS_REFERENCE, RequestMethod.GET);
- return dataFiles.stream().map(FileSliceDTO::toFileSlice);
+ List<FileGroupDTO> fileGroups = executeRequest(requestPath, paramsMap,
+ FILE_GROUP_DTOS_REFERENCE, RequestMethod.GET);
+ return DTOUtils.fileGroupDTOsToFileGroups(fileGroups, metaClient);
} catch (IOException e) {
throw new HoodieRemoteException(e);
}
@@ -449,73 +396,37 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
@Override
public Stream<HoodieFileGroup> getAllFileGroups(String partitionPath) {
Map<String, String> paramsMap = getParamsWithPartitionPath(partitionPath);
- try {
- List<FileGroupDTO> fileGroups = executeRequest(ALL_FILEGROUPS_FOR_PARTITION_URL, paramsMap,
- FILE_GROUP_DTOS_REFERENCE, RequestMethod.GET);
- return DTOUtils.fileGroupDTOsToFileGroups(fileGroups, metaClient);
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ return getAllFileGroupsForPartitionFromParams(ALL_FILEGROUPS_FOR_PARTITION_URL, paramsMap);
}
@Override
public Stream<HoodieFileGroup> getAllFileGroupsStateless(String partitionPath) {
Map<String, String> paramsMap = getParamsWithPartitionPath(partitionPath);
- try {
- List<FileGroupDTO> fileGroups = executeRequest(ALL_FILEGROUPS_FOR_PARTITION_STATELESS_URL, paramsMap,
- new TypeReference<List<FileGroupDTO>>() {}, RequestMethod.GET);
- return DTOUtils.fileGroupDTOsToFileGroups(fileGroups, metaClient);
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ return getAllFileGroupsForPartitionFromParams(ALL_FILEGROUPS_FOR_PARTITION_STATELESS_URL, paramsMap);
}
@Override
public Stream<HoodieFileGroup> getReplacedFileGroupsBeforeOrOn(String maxCommitTime, String partitionPath) {
Map<String, String> paramsMap = getParamsWithAdditionalParam(partitionPath, MAX_INSTANT_PARAM, maxCommitTime);
- try {
- List<FileGroupDTO> fileGroups = executeRequest(ALL_REPLACED_FILEGROUPS_BEFORE_OR_ON, paramsMap,
- FILE_GROUP_DTOS_REFERENCE, RequestMethod.GET);
- return DTOUtils.fileGroupDTOsToFileGroups(fileGroups, metaClient);
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ return getAllFileGroupsForPartitionFromParams(ALL_REPLACED_FILEGROUPS_BEFORE_OR_ON_URL, paramsMap);
}
@Override
public Stream<HoodieFileGroup> getReplacedFileGroupsBefore(String maxCommitTime, String partitionPath) {
Map<String, String> paramsMap = getParamsWithAdditionalParam(partitionPath, MAX_INSTANT_PARAM, maxCommitTime);
- try {
- List<FileGroupDTO> fileGroups = executeRequest(ALL_REPLACED_FILEGROUPS_BEFORE, paramsMap,
- FILE_GROUP_DTOS_REFERENCE, RequestMethod.GET);
- return DTOUtils.fileGroupDTOsToFileGroups(fileGroups, metaClient);
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ return getAllFileGroupsForPartitionFromParams(ALL_REPLACED_FILEGROUPS_BEFORE_URL, paramsMap);
}
@Override
public Stream<HoodieFileGroup> getReplacedFileGroupsAfterOrOn(String minCommitTime, String partitionPath) {
Map<String, String> paramsMap = getParamsWithAdditionalParam(partitionPath, MIN_INSTANT_PARAM, minCommitTime);
- try {
- List<FileGroupDTO> fileGroups = executeRequest(ALL_REPLACED_FILEGROUPS_AFTER_OR_ON, paramsMap,
- FILE_GROUP_DTOS_REFERENCE, RequestMethod.GET);
- return DTOUtils.fileGroupDTOsToFileGroups(fileGroups, metaClient);
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ return getAllFileGroupsForPartitionFromParams(ALL_REPLACED_FILEGROUPS_AFTER_OR_ON_URL, paramsMap);
}
@Override
public Stream<HoodieFileGroup> getAllReplacedFileGroups(String partitionPath) {
Map<String, String> paramsMap = getParamsWithPartitionPath(partitionPath);
- try {
- List<FileGroupDTO> fileGroups = executeRequest(ALL_REPLACED_FILEGROUPS_PARTITION, paramsMap,
- FILE_GROUP_DTOS_REFERENCE, RequestMethod.GET);
- return DTOUtils.fileGroupDTOsToFileGroups(fileGroups, metaClient);
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ return getAllFileGroupsForPartitionFromParams(ALL_REPLACED_FILEGROUPS_PARTITION_URL, paramsMap);
}
public boolean refresh() {
@@ -523,38 +434,40 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
try {
// refresh the local timeline first.
this.timeline = metaClient.reloadActiveTimeline().filterCompletedAndCompactionInstants();
- return executeRequest(REFRESH_TABLE, paramsMap, BOOLEAN_TYPE_REFERENCE, RequestMethod.POST);
+ return executeRequest(REFRESH_TABLE_URL, paramsMap, BOOLEAN_TYPE_REFERENCE, RequestMethod.POST);
} catch (IOException e) {
throw new HoodieRemoteException(e);
}
}
- @Override
- public void loadAllPartitions() {
- Map<String, String> paramsMap = getParams();
+ private void loadPartitions(String requestPath, Map<String, String> paramsMap) {
try {
- executeRequest(LOAD_ALL_PARTITIONS_URL, paramsMap, BOOLEAN_TYPE_REFERENCE, RequestMethod.POST);
+ executeRequest(requestPath, paramsMap, BOOLEAN_TYPE_REFERENCE, RequestMethod.POST);
} catch (IOException e) {
throw new HoodieRemoteException(e);
}
}
+ @Override
+ public void loadAllPartitions() {
+ Map<String, String> paramsMap = getParams();
+ loadPartitions(LOAD_ALL_PARTITIONS_URL, paramsMap);
+ }
+
@Override
public void loadPartitions(List<String> partitionPaths) {
+ Map<String, String> paramsMap = getParams();
try {
- Map<String, String> paramsMap = getParams();
paramsMap.put(PARTITIONS_PARAM, OBJECT_MAPPER.writeValueAsString(partitionPaths));
- executeRequest(LOAD_PARTITIONS_URL, paramsMap, BOOLEAN_TYPE_REFERENCE, RequestMethod.POST);
- } catch (IOException e) {
+ } catch (JsonProcessingException e) {
throw new HoodieRemoteException(e);
}
+ loadPartitions(LOAD_PARTITIONS_URL, paramsMap);
}
- @Override
- public Stream<Pair<String, CompactionOperation>> getPendingCompactionOperations() {
- Map<String, String> paramsMap = getParams();
+ private Stream<Pair<String, CompactionOperation>> getPendingCompactionOperations(String requestPath, Map<String, String> paramsMap) {
try {
- List<CompactionOpDTO> dtos = executeRequest(PENDING_COMPACTION_OPS, paramsMap,
+ List<CompactionOpDTO> dtos = executeRequest(requestPath, paramsMap,
COMPACTION_OP_DTOS_REFERENCE, RequestMethod.GET);
return dtos.stream().map(CompactionOpDTO::toCompactionOperation);
} catch (IOException e) {
@@ -562,23 +475,23 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
}
}
+ @Override
+ public Stream<Pair<String, CompactionOperation>> getPendingCompactionOperations() {
+ Map<String, String> paramsMap = getParams();
+ return getPendingCompactionOperations(PENDING_COMPACTION_OPS_URL, paramsMap);
+ }
+
@Override
public Stream<Pair<String, CompactionOperation>> getPendingLogCompactionOperations() {
Map<String, String> paramsMap = getParams();
- try {
- List<CompactionOpDTO> dtos = executeRequest(PENDING_LOG_COMPACTION_OPS, paramsMap,
- COMPACTION_OP_DTOS_REFERENCE, RequestMethod.GET);
- return dtos.stream().map(CompactionOpDTO::toCompactionOperation);
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ return getPendingCompactionOperations(PENDING_LOG_COMPACTION_OPS_URL, paramsMap);
}
@Override
public Stream<Pair<HoodieFileGroupId, HoodieInstant>> getFileGroupsInPendingClustering() {
Map<String, String> paramsMap = getParams();
try {
- List<ClusteringOpDTO> dtos = executeRequest(PENDING_CLUSTERING_FILEGROUPS, paramsMap,
+ List<ClusteringOpDTO> dtos = executeRequest(PENDING_CLUSTERING_FILEGROUPS_URL, paramsMap,
CLUSTERING_OP_DTOS_REFERENCE, RequestMethod.GET);
return dtos.stream().map(ClusteringOpDTO::toClusteringOperation);
} catch (IOException e) {
@@ -586,22 +499,11 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
}
}
- @Override
- public void close() {
- closed = true;
- }
-
- @Override
- public void reset() {
- refresh();
- }
-
@Override
public Option<HoodieInstant> getLastInstant() {
Map<String, String> paramsMap = getParams();
try {
- List<InstantDTO> instants =
- executeRequest(LAST_INSTANT, paramsMap, INSTANT_DTOS_REFERENCE, RequestMethod.GET);
+ List<InstantDTO> instants = executeRequest(LAST_INSTANT_URL, paramsMap, INSTANT_DTOS_REFERENCE, RequestMethod.GET);
return Option.fromJavaOptional(instants.stream().map(InstantDTO::toInstant).findFirst());
} catch (IOException e) {
throw new HoodieRemoteException(e);
@@ -612,31 +514,26 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView,
public HoodieTimeline getTimeline() {
Map<String, String> paramsMap = getParams();
try {
- TimelineDTO timeline =
- executeRequest(TIMELINE, paramsMap, TIMELINE_DTO_REFERENCE, RequestMethod.GET);
- return TimelineDTO.toTimeline(timeline, metaClient);
+ TimelineDTO timelineDto = executeRequest(TIMELINE_URL, paramsMap, TIMELINE_DTO_REFERENCE, RequestMethod.GET);
+ return TimelineDTO.toTimeline(timelineDto, metaClient);
} catch (IOException e) {
throw new HoodieRemoteException(e);
}
}
@Override
- public void sync() {
+ public void close() {
+ closed = true;
+ }
+
+ @Override
+ public void reset() {
refresh();
}
@Override
- public Option<HoodieBaseFile> getLatestBaseFile(String partitionPath, String fileId) {
- Map<String, String> paramsMap = getParamsWithAdditionalParam(partitionPath, FILEID_PARAM, fileId);
- try {
- List<BaseFileDTO> dataFiles = executeRequest(LATEST_PARTITION_DATA_FILE_URL, paramsMap,
- BASE_FILE_DTOS_REFERENCE, RequestMethod.GET);
- return Option.fromJavaOptional(dataFiles.stream()
- .map(BaseFileDTO::toHoodieBaseFile)
- .findFirst());
- } catch (IOException e) {
- throw new HoodieRemoteException(e);
- }
+ public void sync() {
+ refresh();
}
private Response get(int timeoutMs, String url, RequestMethod method) throws IOException {
diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/RequestHandler.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/RequestHandler.java
index 009a7bf848b..1a1ac5563ac 100644
--- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/RequestHandler.java
+++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/RequestHandler.java
@@ -129,15 +129,50 @@ public class RequestHandler {
metricsRegistry.add("WRITE_VALUE_CNT", 1);
metricsRegistry.add("WRITE_VALUE_TIME", jsonifyTime);
if (logger.isDebugEnabled()) {
- logger.debug("Jsonify TimeTaken=" + jsonifyTime);
+ logger.debug("Jsonify TimeTaken={}", jsonifyTime);
}
return result;
}
- private static boolean isRefreshCheckDisabledInQuery(Context ctxt) {
- return Boolean.parseBoolean(ctxt.queryParam(RemoteHoodieTableFileSystemView.REFRESH_OFF));
+ private static String getBasePathParam(Context ctx) {
+ return ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid"));
}
+ private static String getPartitionParam(Context ctx) {
+ return ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault("");
+ }
+
+ private static String getFileIdParam(Context ctx) {
+ return ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.FILEID_PARAM, String.class).getOrThrow(e -> new HoodieException("FILEID is invalid"));
+ }
+
+ private static List<String> getInstantsParam(Context ctx) {
+ return Arrays.asList(ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.INSTANTS_PARAM, String.class).getOrThrow(e -> new HoodieException("INSTANTS_PARAM is invalid"))
+ .split(RemoteHoodieTableFileSystemView.MULTI_VALUE_SEPARATOR));
+ }
+
+ private static String getMaxInstantParamMandatory(Context ctx) {
+ return ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.MAX_INSTANT_PARAM, String.class).getOrThrow(e -> new HoodieException("MAX_INSTANT_PARAM is invalid"));
+ }
+
+ private static String getMaxInstantParamOptional(Context ctx) {
+ return ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.MAX_INSTANT_PARAM, String.class).getOrDefault("");
+ }
+
+ private static String getMinInstantParam(Context ctx) {
+ return ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.MIN_INSTANT_PARAM, String.class).getOrDefault("");
+ }
+
+ private static String getMarkerDirParam(Context ctx) {
+ return ctx.queryParamAsClass(MarkerOperation.MARKER_DIR_PATH_PARAM, String.class).getOrDefault("");
+ }
+
+ private static boolean getIncludeFilesInPendingCompactionParam(Context ctx) {
+ return Boolean.parseBoolean(
+ ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.INCLUDE_FILES_IN_PENDING_COMPACTION_PARAM, String.class)
+ .getOrThrow(e -> new HoodieException("INCLUDE_FILES_IN_PENDING_COMPACTION_PARAM is invalid")));
+ }
+
public void register() {
registerDataFilesAPI();
registerFileSlicesAPI();
@@ -153,59 +188,6 @@ public class RequestHandler {
}
}
- /**
- * Determines if local view of table's timeline is behind that of client's view.
- */
- private boolean isLocalViewBehind(Context ctx) {
- String basePath = ctx.queryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM);
- String lastKnownInstantFromClient =
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.LAST_INSTANT_TS, String.class).getOrDefault(HoodieTimeline.INVALID_INSTANT_TS);
- String timelineHashFromClient = ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.TIMELINE_HASH, String.class).getOrDefault("");
- HoodieTimeline localTimeline =
- viewManager.getFileSystemView(basePath).getTimeline().filterCompletedOrMajorOrMinorCompactionInstants();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Client [ LastTs=" + lastKnownInstantFromClient + ", TimelineHash=" + timelineHashFromClient
- + "], localTimeline=" + localTimeline.getInstants());
- }
-
- if ((!localTimeline.getInstantsAsStream().findAny().isPresent())
- && HoodieTimeline.INVALID_INSTANT_TS.equals(lastKnownInstantFromClient)) {
- return false;
- }
-
- String localTimelineHash = localTimeline.getTimelineHash();
- // refresh if timeline hash mismatches
- if (!localTimelineHash.equals(timelineHashFromClient)) {
- return true;
- }
-
- // As a safety check, even if hash is same, ensure instant is present
- return !localTimeline.containsOrBeforeTimelineStarts(lastKnownInstantFromClient);
- }
-
- /**
- * Syncs data-set view if local view is behind.
- */
- private boolean syncIfLocalViewBehind(Context ctx) {
- String basePath = ctx.queryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM);
- SyncableFileSystemView view = viewManager.getFileSystemView(basePath);
- synchronized (view) {
- if (isLocalViewBehind(ctx)) {
-
- String lastKnownInstantFromClient = ctx.queryParamAsClass(
- RemoteHoodieTableFileSystemView.LAST_INSTANT_TS, String.class)
- .getOrDefault(HoodieTimeline.INVALID_INSTANT_TS);
- HoodieTimeline localTimeline = viewManager.getFileSystemView(basePath).getTimeline();
- LOG.info("Syncing view as client passed last known instant " + lastKnownInstantFromClient
- + " as last known instant but server has the following last instant on timeline :"
- + localTimeline.lastInstant());
- view.sync();
- return true;
- }
- }
- return false;
- }
-
private void writeValueAsString(Context ctx, Object obj) throws JsonProcessingException {
if (timelineServiceConfig.async) {
writeValueAsStringAsync(ctx, obj);
@@ -233,15 +215,15 @@ public class RequestHandler {
* Register Timeline API calls.
*/
private void registerTimelineAPI() {
- app.get(RemoteHoodieTableFileSystemView.LAST_INSTANT, new ViewHandler(ctx -> {
+ app.get(RemoteHoodieTableFileSystemView.LAST_INSTANT_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LAST_INSTANT", 1);
- List<InstantDTO> dtos = instantHandler.getLastInstant(ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).get());
+ List<InstantDTO> dtos = instantHandler.getLastInstant(getBasePathParam(ctx));
writeValueAsString(ctx, dtos);
}, false));
- app.get(RemoteHoodieTableFileSystemView.TIMELINE, new ViewHandler(ctx -> {
+ app.get(RemoteHoodieTableFileSystemView.TIMELINE_URL, new ViewHandler(ctx -> {
metricsRegistry.add("TIMELINE", 1);
- TimelineDTO dto = instantHandler.getTimeline(ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).get());
+ TimelineDTO dto = instantHandler.getTimeline(getBasePathParam(ctx));
writeValueAsString(ctx, dto);
}, false));
}
@@ -253,68 +235,66 @@ public class RequestHandler {
app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_DATA_FILES_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LATEST_PARTITION_DATA_FILES", 1);
List<BaseFileDTO> dtos = dataFileHandler.getLatestDataFiles(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""));
-
+ getBasePathParam(ctx),
+ getPartitionParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_DATA_FILE_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LATEST_PARTITION_DATA_FILE", 1);
List<BaseFileDTO> dtos = dataFileHandler.getLatestDataFile(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.FILEID_PARAM, String.class).getOrThrow(e -> new HoodieException("FILEID is invalid")));
+ getBasePathParam(ctx),
+ getPartitionParam(ctx),
+ getFileIdParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
- app.get(RemoteHoodieTableFileSystemView.LATEST_ALL_DATA_FILES, new ViewHandler(ctx -> {
+ app.get(RemoteHoodieTableFileSystemView.LATEST_ALL_DATA_FILES_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LATEST_ALL_DATA_FILES", 1);
- List<BaseFileDTO> dtos = dataFileHandler.getLatestDataFiles(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")));
+ List<BaseFileDTO> dtos = dataFileHandler.getLatestDataFiles(getBasePathParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.LATEST_DATA_FILES_BEFORE_ON_INSTANT_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LATEST_DATA_FILES_BEFORE_ON_INSTANT", 1);
List<BaseFileDTO> dtos = dataFileHandler.getLatestDataFilesBeforeOrOn(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.MAX_INSTANT_PARAM, String.class).getOrThrow(e -> new HoodieException("MAX_INSTANT_PARAM is invalid")));
+ getBasePathParam(ctx),
+ getPartitionParam(ctx),
+ getMaxInstantParamMandatory(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.ALL_LATEST_BASE_FILES_BEFORE_ON_INSTANT_URL, new ViewHandler(ctx -> {
metricsRegistry.add("ALL_LATEST_BASE_FILES_BEFORE_ON_INSTANT", 1);
Map<String, List<BaseFileDTO>> dtos = dataFileHandler.getAllLatestDataFilesBeforeOrOn(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.MAX_INSTANT_PARAM, String.class).getOrThrow(e -> new HoodieException("MAX_INSTANT_PARAM is invalid")));
+ getBasePathParam(ctx),
+ getMaxInstantParamMandatory(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.LATEST_DATA_FILE_ON_INSTANT_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LATEST_DATA_FILE_ON_INSTANT", 1);
List<BaseFileDTO> dtos = dataFileHandler.getLatestDataFileOn(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""),
+ getBasePathParam(ctx),
+ getPartitionParam(ctx),
ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.INSTANT_PARAM, String.class).get(),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.FILEID_PARAM, String.class).getOrThrow(e -> new HoodieException("FILEID is invalid")));
+ getFileIdParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
- app.get(RemoteHoodieTableFileSystemView.ALL_DATA_FILES, new ViewHandler(ctx -> {
+ app.get(RemoteHoodieTableFileSystemView.ALL_DATA_FILES_URL, new ViewHandler(ctx -> {
metricsRegistry.add("ALL_DATA_FILES", 1);
List<BaseFileDTO> dtos = dataFileHandler.getAllDataFiles(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""));
+ getBasePathParam(ctx),
+ getPartitionParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.LATEST_DATA_FILES_RANGE_INSTANT_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LATEST_DATA_FILES_RANGE_INSTANT", 1);
List<BaseFileDTO> dtos = dataFileHandler.getLatestDataFilesInRange(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- Arrays.asList(ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.INSTANTS_PARAM, String.class).getOrThrow(e -> new HoodieException("INSTANTS_PARAM is invalid")).split(",")));
+ getBasePathParam(ctx),
+ getInstantsParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
}
@@ -326,121 +306,116 @@ public class RequestHandler {
app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_SLICES_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LATEST_PARTITION_SLICES", 1);
List<FileSliceDTO> dtos = sliceHandler.getLatestFileSlices(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""));
+ getBasePathParam(ctx),
+ getPartitionParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_SLICES_STATELESS_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LATEST_PARTITION_SLICES_STATELESS", 1);
List<FileSliceDTO> dtos = sliceHandler.getLatestFileSlicesStateless(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""));
+ getBasePathParam(ctx),
+ getPartitionParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_SLICE_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LATEST_PARTITION_SLICE", 1);
List<FileSliceDTO> dtos = sliceHandler.getLatestFileSlice(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.FILEID_PARAM, String.class).getOrThrow(e -> new HoodieException("FILEID is invalid")));
+ getBasePathParam(ctx),
+ getPartitionParam(ctx),
+ getFileIdParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_UNCOMPACTED_SLICES_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LATEST_PARTITION_UNCOMPACTED_SLICES", 1);
List<FileSliceDTO> dtos = sliceHandler.getLatestUnCompactedFileSlices(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""));
+ getBasePathParam(ctx),
+ getPartitionParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.ALL_SLICES_URL, new ViewHandler(ctx -> {
metricsRegistry.add("ALL_SLICES", 1);
List<FileSliceDTO> dtos = sliceHandler.getAllFileSlices(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""));
+ getBasePathParam(ctx),
+ getPartitionParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.LATEST_SLICES_RANGE_INSTANT_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LATEST_SLICE_RANGE_INSTANT", 1);
List<FileSliceDTO> dtos = sliceHandler.getLatestFileSliceInRange(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- Arrays.asList(ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.INSTANTS_PARAM, String.class).getOrThrow(e -> new HoodieException("INSTANTS_PARAM is invalid")).split(",")));
+ getBasePathParam(ctx),
+ getInstantsParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.LATEST_SLICES_MERGED_BEFORE_ON_INSTANT_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LATEST_SLICES_MERGED_BEFORE_ON_INSTANT", 1);
List<FileSliceDTO> dtos = sliceHandler.getLatestMergedFileSlicesBeforeOrOn(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.MAX_INSTANT_PARAM, String.class).getOrThrow(e -> new HoodieException("MAX_INSTANT_PARAM is invalid")));
+ getBasePathParam(ctx),
+ getPartitionParam(ctx),
+ getMaxInstantParamMandatory(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.LATEST_SLICES_BEFORE_ON_INSTANT_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LATEST_SLICES_BEFORE_ON_INSTANT", 1);
List<FileSliceDTO> dtos = sliceHandler.getLatestFileSlicesBeforeOrOn(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.MAX_INSTANT_PARAM, String.class).getOrThrow(e -> new HoodieException("MAX_INSTANT_PARAM is invalid")),
- Boolean.parseBoolean(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.INCLUDE_FILES_IN_PENDING_COMPACTION_PARAM, String.class)
- .getOrThrow(e -> new HoodieException("INCLUDE_FILES_IN_PENDING_COMPACTION_PARAM is invalid"))));
+ getBasePathParam(ctx),
+ getPartitionParam(ctx),
+ getMaxInstantParamMandatory(ctx),
+ getIncludeFilesInPendingCompactionParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.ALL_LATEST_SLICES_BEFORE_ON_INSTANT_URL, new ViewHandler(ctx -> {
metricsRegistry.add("ALL_LATEST_SLICES_BEFORE_ON_INSTANT", 1);
Map<String, List<FileSliceDTO>> dtos = sliceHandler.getAllLatestFileSlicesBeforeOrOn(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.MAX_INSTANT_PARAM, String.class).getOrThrow(e -> new HoodieException("MAX_INSTANT_PARAM is invalid")));
+ getBasePathParam(ctx),
+ getMaxInstantParamMandatory(ctx));
writeValueAsString(ctx, dtos);
}, true));
- app.get(RemoteHoodieTableFileSystemView.PENDING_COMPACTION_OPS, new ViewHandler(ctx -> {
+ app.get(RemoteHoodieTableFileSystemView.PENDING_COMPACTION_OPS_URL, new ViewHandler(ctx -> {
metricsRegistry.add("PEDING_COMPACTION_OPS", 1);
- List<CompactionOpDTO> dtos = sliceHandler.getPendingCompactionOperations(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")));
+ List<CompactionOpDTO> dtos = sliceHandler.getPendingCompactionOperations(getBasePathParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
- app.get(RemoteHoodieTableFileSystemView.PENDING_LOG_COMPACTION_OPS, new ViewHandler(ctx -> {
+ app.get(RemoteHoodieTableFileSystemView.PENDING_LOG_COMPACTION_OPS_URL, new ViewHandler(ctx -> {
metricsRegistry.add("PEDING_LOG_COMPACTION_OPS", 1);
- List<CompactionOpDTO> dtos = sliceHandler.getPendingLogCompactionOperations(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")));
+ List<CompactionOpDTO> dtos = sliceHandler.getPendingLogCompactionOperations(getBasePathParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.ALL_FILEGROUPS_FOR_PARTITION_URL, new ViewHandler(ctx -> {
metricsRegistry.add("ALL_FILEGROUPS_FOR_PARTITION", 1);
List<FileGroupDTO> dtos = sliceHandler.getAllFileGroups(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""));
+ getBasePathParam(ctx),
+ getPartitionParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
app.get(RemoteHoodieTableFileSystemView.ALL_FILEGROUPS_FOR_PARTITION_STATELESS_URL, new ViewHandler(ctx -> {
metricsRegistry.add("ALL_FILEGROUPS_FOR_PARTITION_STATELESS", 1);
List<FileGroupDTO> dtos = sliceHandler.getAllFileGroupsStateless(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""));
+ getBasePathParam(ctx),
+ getPartitionParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
- app.post(RemoteHoodieTableFileSystemView.REFRESH_TABLE, new ViewHandler(ctx -> {
+ app.post(RemoteHoodieTableFileSystemView.REFRESH_TABLE_URL, new ViewHandler(ctx -> {
metricsRegistry.add("REFRESH_TABLE", 1);
- boolean success = sliceHandler
- .refreshTable(ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")));
+ boolean success = sliceHandler.refreshTable(getBasePathParam(ctx));
writeValueAsString(ctx, success);
}, false));
app.post(RemoteHoodieTableFileSystemView.LOAD_PARTITIONS_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LOAD_PARTITIONS", 1);
- String basePath = ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid"));
+ String basePath = getBasePathParam(ctx);
try {
List<String> partitionPaths = OBJECT_MAPPER.readValue(ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITIONS_PARAM, String.class)
.getOrThrow(e -> new HoodieException("Partitions param is invalid")), LIST_TYPE_REFERENCE);
@@ -453,50 +428,48 @@ public class RequestHandler {
app.post(RemoteHoodieTableFileSystemView.LOAD_ALL_PARTITIONS_URL, new ViewHandler(ctx -> {
metricsRegistry.add("LOAD_ALL_PARTITIONS", 1);
- boolean success = sliceHandler
- .loadAllPartitions(ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")));
+ boolean success = sliceHandler.loadAllPartitions(getBasePathParam(ctx));
writeValueAsString(ctx, success);
}, false));
- app.get(RemoteHoodieTableFileSystemView.ALL_REPLACED_FILEGROUPS_BEFORE_OR_ON, new ViewHandler(ctx -> {
+ app.get(RemoteHoodieTableFileSystemView.ALL_REPLACED_FILEGROUPS_BEFORE_OR_ON_URL, new ViewHandler(ctx -> {
metricsRegistry.add("ALL_REPLACED_FILEGROUPS_BEFORE_OR_ON", 1);
List<FileGroupDTO> dtos = sliceHandler.getReplacedFileGroupsBeforeOrOn(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.MAX_INSTANT_PARAM, String.class).getOrDefault(""),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""));
+ getBasePathParam(ctx),
+ getMaxInstantParamOptional(ctx),
+ getPartitionParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
- app.get(RemoteHoodieTableFileSystemView.ALL_REPLACED_FILEGROUPS_BEFORE, new ViewHandler(ctx -> {
+ app.get(RemoteHoodieTableFileSystemView.ALL_REPLACED_FILEGROUPS_BEFORE_URL, new ViewHandler(ctx -> {
metricsRegistry.add("ALL_REPLACED_FILEGROUPS_BEFORE", 1);
List<FileGroupDTO> dtos = sliceHandler.getReplacedFileGroupsBefore(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.MAX_INSTANT_PARAM, String.class).getOrDefault(""),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""));
+ getBasePathParam(ctx),
+ getMaxInstantParamOptional(ctx),
+ getPartitionParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
- app.get(RemoteHoodieTableFileSystemView.ALL_REPLACED_FILEGROUPS_AFTER_OR_ON, new ViewHandler(ctx -> {
+ app.get(RemoteHoodieTableFileSystemView.ALL_REPLACED_FILEGROUPS_AFTER_OR_ON_URL, new ViewHandler(ctx -> {
metricsRegistry.add("ALL_REPLACED_FILEGROUPS_AFTER_OR_ON", 1);
List<FileGroupDTO> dtos = sliceHandler.getReplacedFileGroupsAfterOrOn(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.MIN_INSTANT_PARAM, String.class).getOrDefault(""),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""));
+ getBasePathParam(ctx),
+ getMinInstantParam(ctx),
+ getPartitionParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
- app.get(RemoteHoodieTableFileSystemView.ALL_REPLACED_FILEGROUPS_PARTITION, new ViewHandler(ctx -> {
+ app.get(RemoteHoodieTableFileSystemView.ALL_REPLACED_FILEGROUPS_PARTITION_URL, new ViewHandler(ctx -> {
metricsRegistry.add("ALL_REPLACED_FILEGROUPS_PARTITION", 1);
List<FileGroupDTO> dtos = sliceHandler.getAllReplacedFileGroups(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")),
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.PARTITION_PARAM, String.class).getOrDefault(""));
+ getBasePathParam(ctx),
+ getPartitionParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
- app.get(RemoteHoodieTableFileSystemView.PENDING_CLUSTERING_FILEGROUPS, new ViewHandler(ctx -> {
+ app.get(RemoteHoodieTableFileSystemView.PENDING_CLUSTERING_FILEGROUPS_URL, new ViewHandler(ctx -> {
metricsRegistry.add("PENDING_CLUSTERING_FILEGROUPS", 1);
- List<ClusteringOpDTO> dtos = sliceHandler.getFileGroupsInPendingClustering(
- ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.BASEPATH_PARAM, String.class).getOrThrow(e -> new HoodieException("Basepath is invalid")));
+ List<ClusteringOpDTO> dtos = sliceHandler.getFileGroupsInPendingClustering(getBasePathParam(ctx));
writeValueAsString(ctx, dtos);
}, true));
}
@@ -504,15 +477,13 @@ public class RequestHandler {
private void registerMarkerAPI() {
app.get(MarkerOperation.ALL_MARKERS_URL, new ViewHandler(ctx -> {
metricsRegistry.add("ALL_MARKERS", 1);
- Set<String> markers = markerHandler.getAllMarkers(
- ctx.queryParamAsClass(MarkerOperation.MARKER_DIR_PATH_PARAM, String.class).getOrDefault(""));
+ Set<String> markers = markerHandler.getAllMarkers(getMarkerDirParam(ctx));
writeValueAsString(ctx, markers);
}, false));
app.get(MarkerOperation.CREATE_AND_MERGE_MARKERS_URL, new ViewHandler(ctx -> {
metricsRegistry.add("CREATE_AND_MERGE_MARKERS", 1);
- Set<String> markers = markerHandler.getCreateAndMergeMarkers(
- ctx.queryParamAsClass(MarkerOperation.MARKER_DIR_PATH_PARAM, String.class).getOrDefault(""));
+ Set<String> markers = markerHandler.getCreateAndMergeMarkers(getMarkerDirParam(ctx));
writeValueAsString(ctx, markers);
}, false));
@@ -525,8 +496,7 @@ public class RequestHandler {
app.get(MarkerOperation.MARKERS_DIR_EXISTS_URL, new ViewHandler(ctx -> {
metricsRegistry.add("MARKERS_DIR_EXISTS", 1);
- boolean exist = markerHandler.doesMarkerDirExist(
- ctx.queryParamAsClass(MarkerOperation.MARKER_DIR_PATH_PARAM, String.class).getOrDefault(""));
+ boolean exist = markerHandler.doesMarkerDirExist(getMarkerDirParam(ctx));
writeValueAsString(ctx, exist);
}, false));
@@ -534,30 +504,18 @@ public class RequestHandler {
metricsRegistry.add("CREATE_MARKER", 1);
ctx.future(markerHandler.createMarker(
ctx,
- ctx.queryParamAsClass(MarkerOperation.MARKER_DIR_PATH_PARAM, String.class).getOrDefault(""),
+ getMarkerDirParam(ctx),
ctx.queryParamAsClass(MarkerOperation.MARKER_NAME_PARAM, String.class).getOrDefault(""),
ctx.queryParamAsClass(MarkerOperation.MARKER_BASEPATH_PARAM, String.class).getOrDefault("")));
}, false));
app.post(MarkerOperation.DELETE_MARKER_DIR_URL, new ViewHandler(ctx -> {
metricsRegistry.add("DELETE_MARKER_DIR", 1);
- boolean success = markerHandler.deleteMarkers(
- ctx.queryParamAsClass(MarkerOperation.MARKER_DIR_PATH_PARAM, String.class).getOrDefault(""));
+ boolean success = markerHandler.deleteMarkers(getMarkerDirParam(ctx));
writeValueAsString(ctx, success);
}, false));
}
- /**
- * Determine whether to throw an exception when local view of table's timeline is behind that of client's view.
- */
- private boolean shouldThrowExceptionIfLocalViewBehind(HoodieTimeline localTimeline, String timelineHashFromClient) {
- Option<HoodieInstant> lastInstant = localTimeline.lastInstant();
- // When performing async clean, we may have one more .clean.completed after lastInstantTs.
- // In this case, we do not need to throw an exception.
- return !lastInstant.isPresent() || !lastInstant.get().getAction().equals(HoodieTimeline.CLEAN_ACTION)
- || !localTimeline.findInstantsBefore(lastInstant.get().getTimestamp()).getTimelineHash().equals(timelineHashFromClient);
- }
-
/**
* Used for logging and performing refresh check.
*/
@@ -604,16 +562,13 @@ public class RequestHandler {
if (refreshCheck) {
long beginFinalCheck = System.currentTimeMillis();
if (isLocalViewBehind(context)) {
- String lastKnownInstantFromClient = context.queryParamAsClass(RemoteHoodieTableFileSystemView.LAST_INSTANT_TS, String.class).getOrDefault(HoodieTimeline.INVALID_INSTANT_TS);
- String timelineHashFromClient = context.queryParamAsClass(RemoteHoodieTableFileSystemView.TIMELINE_HASH, String.class).getOrDefault("");
+ String lastKnownInstantFromClient = getLastInstantTsParam(context);
+ String timelineHashFromClient = getTimelineHashParam(context);
HoodieTimeline localTimeline =
viewManager.getFileSystemView(context.queryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM)).getTimeline();
if (shouldThrowExceptionIfLocalViewBehind(localTimeline, timelineHashFromClient)) {
- String errMsg =
- "Last known instant from client was "
- + lastKnownInstantFromClient
- + " but server has the following timeline "
- + localTimeline.getInstants();
+ String errMsg = String.format("Last known instant from client was %s but server has the following timeline %s",
+ lastKnownInstantFromClient, localTimeline.getInstants());
throw new BadRequestResponse(errMsg);
}
}
@@ -623,9 +578,9 @@ public class RequestHandler {
} catch (RuntimeException re) {
success = false;
if (re instanceof BadRequestResponse) {
- LOG.warn("Bad request response due to client view behind server view. " + re.getMessage());
+ LOG.warn("Bad request response due to client view behind server view. {}", re.getMessage());
} else {
- LOG.error("Got runtime exception servicing request " + context.queryString(), re);
+ LOG.error(String.format("Got runtime exception servicing request %s", context.queryString()), re);
}
throw re;
} finally {
@@ -637,14 +592,85 @@ public class RequestHandler {
metricsRegistry.add("TOTAL_CHECK_TIME", finalCheckTimeTaken);
metricsRegistry.add("TOTAL_API_CALLS", 1);
- LOG.debug(String.format(
- "TimeTakenMillis[Total=%d, Refresh=%d, handle=%d, Check=%d], "
- + "Success=%s, Query=%s, Host=%s, synced=%s",
- timeTakenMillis, refreshCheckTimeTaken, handleTimeTaken, finalCheckTimeTaken, success,
- context.queryString(), context.host(), synced));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("TimeTakenMillis[Total={}, Refresh={}, handle={}, Check={}], Success={}, Query={}, Host={}, synced={}",
+ timeTakenMillis, refreshCheckTimeTaken, handleTimeTaken, finalCheckTimeTaken, success, context.queryString(), context.host(), synced);
+ }
}
return null;
});
}
+
+ /**
+ * Determines if local view of table's timeline is behind that of client's view.
+ */
+ private boolean isLocalViewBehind(Context ctx) {
+ String basePath = ctx.queryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM);
+ String lastKnownInstantFromClient = getLastInstantTsParam(ctx);
+ String timelineHashFromClient = getTimelineHashParam(ctx);
+ HoodieTimeline localTimeline =
+ viewManager.getFileSystemView(basePath).getTimeline().filterCompletedOrMajorOrMinorCompactionInstants();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Client [ LastTs={}, TimelineHash={}], localTimeline={}",lastKnownInstantFromClient, timelineHashFromClient, localTimeline.getInstants());
+ }
+
+ if ((!localTimeline.getInstantsAsStream().findAny().isPresent())
+ && HoodieTimeline.INVALID_INSTANT_TS.equals(lastKnownInstantFromClient)) {
+ return false;
+ }
+
+ String localTimelineHash = localTimeline.getTimelineHash();
+ // refresh if timeline hash mismatches
+ if (!localTimelineHash.equals(timelineHashFromClient)) {
+ return true;
+ }
+
+ // As a safety check, even if hash is same, ensure instant is present
+ return !localTimeline.containsOrBeforeTimelineStarts(lastKnownInstantFromClient);
+ }
+
+ /**
+ * Syncs data-set view if local view is behind.
+ */
+ private boolean syncIfLocalViewBehind(Context ctx) {
+ String basePath = ctx.queryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM);
+ SyncableFileSystemView view = viewManager.getFileSystemView(basePath);
+ synchronized (view) {
+ if (isLocalViewBehind(ctx)) {
+ String lastKnownInstantFromClient = getLastInstantTsParam(ctx);
+ HoodieTimeline localTimeline = viewManager.getFileSystemView(basePath).getTimeline();
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Syncing view as client passed last known instant {} as last known instant but server has the following last instant on timeline: {}",
+ lastKnownInstantFromClient, localTimeline.lastInstant());
+ }
+ view.sync();
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Determine whether to throw an exception when local view of table's timeline is behind that of client's view.
+ */
+ private boolean shouldThrowExceptionIfLocalViewBehind(HoodieTimeline localTimeline, String timelineHashFromClient) {
+ Option<HoodieInstant> lastInstant = localTimeline.lastInstant();
+ // When performing async clean, we may have one more .clean.completed after lastInstantTs.
+ // In this case, we do not need to throw an exception.
+ return !lastInstant.isPresent() || !lastInstant.get().getAction().equals(HoodieTimeline.CLEAN_ACTION)
+ || !localTimeline.findInstantsBefore(lastInstant.get().getTimestamp()).getTimelineHash().equals(timelineHashFromClient);
+ }
+
+ private boolean isRefreshCheckDisabledInQuery(Context ctx) {
+ return Boolean.parseBoolean(ctx.queryParam(RemoteHoodieTableFileSystemView.REFRESH_OFF));
+ }
+
+ private String getLastInstantTsParam(Context ctx) {
+ return ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.LAST_INSTANT_TS, String.class).getOrDefault(HoodieTimeline.INVALID_INSTANT_TS);
+ }
+
+ private String getTimelineHashParam(Context ctx) {
+ return ctx.queryParamAsClass(RemoteHoodieTableFileSystemView.TIMELINE_HASH, String.class).getOrDefault("");
+ }
}
}
(hudi) 12/16: [MINOR] Fix incorrect catch of ClassCastException using HoodieSparkKeyGeneratorFactory (#11062)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 9a2a61c64f2d8cb6dc096c70283bccf40acd7326
Author: Марк Бухнер <66...@users.noreply.github.com>
AuthorDate: Tue Apr 23 16:12:35 2024 +0700
[MINOR] Fix incorrect catch of ClassCastException using HoodieSparkKeyGeneratorFactory (#11062)
---
.../run/strategy/ExecutionStrategyUtil.java | 17 +-----
.../MultipleSparkJobExecutionStrategy.java | 3 +-
.../strategy/SingleSparkJobExecutionStrategy.java | 3 +-
.../apache/hudi/index/SparkHoodieIndexFactory.java | 19 +-----
.../factory/HoodieSparkKeyGeneratorFactory.java | 45 +++++++++-----
.../hudi/table/HoodieSparkCopyOnWriteTable.java | 12 +---
.../SparkBootstrapCommitActionExecutor.java | 9 +--
.../commit/BaseSparkCommitActionExecutor.java | 9 +--
.../SparkFullBootstrapDataProviderBase.java | 70 ++++++++++------------
.../apache/hudi/keygen/TestCustomKeyGenerator.java | 2 +-
.../TestHoodieSparkKeyGeneratorFactory.java | 7 +--
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 2 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 2 +-
.../TestSparkSqlWithCustomKeyGenerator.scala | 4 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 4 +-
15 files changed, 79 insertions(+), 129 deletions(-)
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/ExecutionStrategyUtil.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/ExecutionStrategyUtil.java
index b70eed70090..5fd2cb65d69 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/ExecutionStrategyUtil.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/ExecutionStrategyUtil.java
@@ -20,7 +20,6 @@ package org.apache.hudi.client.clustering.run.strategy;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
-import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecordPayload;
@@ -28,13 +27,10 @@ import org.apache.hudi.common.model.RewriteAvroPayload;
import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.keygen.KeyGenUtils;
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
-import java.io.IOException;
-
public class ExecutionStrategyUtil {
/**
@@ -49,18 +45,7 @@ public class ExecutionStrategyUtil {
HoodieWriteConfig writeConfig) {
GenericRecord record = (GenericRecord) indexedRecord;
- Option<BaseKeyGenerator> keyGeneratorOpt = Option.empty();
-
- if (!writeConfig.populateMetaFields()) {
- try {
- TypedProperties typedProperties = new TypedProperties(writeConfig.getProps());
- keyGeneratorOpt = Option.of((BaseKeyGenerator)
- HoodieSparkKeyGeneratorFactory.createKeyGenerator(typedProperties));
- } catch (IOException e) {
- throw new HoodieIOException(
- "Only BaseKeyGenerators are supported when meta columns are disabled ", e);
- }
- }
+ Option<BaseKeyGenerator> keyGeneratorOpt = HoodieSparkKeyGeneratorFactory.createBaseKeyGenerator(writeConfig);
String key = KeyGenUtils.getRecordKeyFromGenericRecord(record, keyGeneratorOpt);
String partition = KeyGenUtils.getPartitionPathFromGenericRecord(record, keyGeneratorOpt);
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
index 9d8c9318dd2..97edc237b40 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
@@ -359,8 +359,7 @@ public abstract class MultipleSparkJobExecutionStrategy<T>
Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(writeConfig.getSchema()));
HoodieFileReader baseFileReader = getBaseOrBootstrapFileReader(hadoopConf, bootstrapBasePath, partitionFields, clusteringOp);
- Option<BaseKeyGenerator> keyGeneratorOp =
- writeConfig.populateMetaFields() ? Option.empty() : Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(writeConfig.getProps()));
+ Option<BaseKeyGenerator> keyGeneratorOp = HoodieSparkKeyGeneratorFactory.createBaseKeyGenerator(writeConfig);
// NOTE: Record have to be cloned here to make sure if it holds low-level engine-specific
// payload pointing into a shared, mutable (underlying) buffer we get a clean copy of
// it since these records will be shuffled later.
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java
index fa2af5d5b90..6353646a07d 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobExecutionStrategy.java
@@ -148,8 +148,7 @@ public abstract class SingleSparkJobExecutionStrategy<T>
try {
HoodieFileReader baseFileReader = HoodieFileReaderFactory.getReaderFactory(recordType)
.getFileReader(writeConfig, getHoodieTable().getHadoopConf(), new StoragePath(clusteringOp.getDataFilePath()));
- Option<BaseKeyGenerator> keyGeneratorOp =
- writeConfig.populateMetaFields() ? Option.empty() : Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(writeConfig.getProps()));
+ Option<BaseKeyGenerator> keyGeneratorOp = HoodieSparkKeyGeneratorFactory.createBaseKeyGenerator(writeConfig);
// NOTE: Record have to be cloned here to make sure if it holds low-level engine-specific
// payload pointing into a shared, mutable (underlying) buffer we get a clean copy of
// it since these records will be shuffled later.
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/SparkHoodieIndexFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/SparkHoodieIndexFactory.java
index eebaf0f05ba..661152c2d16 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/SparkHoodieIndexFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/SparkHoodieIndexFactory.java
@@ -18,11 +18,8 @@
package org.apache.hudi.index;
-import org.apache.hudi.common.config.TypedProperties;
-import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieIndexException;
import org.apache.hudi.index.bloom.HoodieBloomIndex;
import org.apache.hudi.index.bloom.HoodieGlobalBloomIndex;
@@ -33,11 +30,8 @@ import org.apache.hudi.index.hbase.SparkHoodieHBaseIndex;
import org.apache.hudi.index.inmemory.HoodieInMemoryHashIndex;
import org.apache.hudi.index.simple.HoodieGlobalSimpleIndex;
import org.apache.hudi.index.simple.HoodieSimpleIndex;
-import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
-import java.io.IOException;
-
/**
* A factory to generate Spark {@link HoodieIndex}.
*/
@@ -62,9 +56,9 @@ public final class SparkHoodieIndexFactory {
case GLOBAL_BLOOM:
return new HoodieGlobalBloomIndex(config, SparkHoodieBloomIndexHelper.getInstance());
case SIMPLE:
- return new HoodieSimpleIndex(config, getKeyGeneratorForSimpleIndex(config));
+ return new HoodieSimpleIndex(config, HoodieSparkKeyGeneratorFactory.createBaseKeyGenerator(config));
case GLOBAL_SIMPLE:
- return new HoodieGlobalSimpleIndex(config, getKeyGeneratorForSimpleIndex(config));
+ return new HoodieGlobalSimpleIndex(config, HoodieSparkKeyGeneratorFactory.createBaseKeyGenerator(config));
case BUCKET:
switch (config.getBucketIndexEngineType()) {
case SIMPLE:
@@ -108,13 +102,4 @@ public final class SparkHoodieIndexFactory {
return createIndex(config).isGlobal();
}
}
-
- private static Option<BaseKeyGenerator> getKeyGeneratorForSimpleIndex(HoodieWriteConfig config) {
- try {
- return config.populateMetaFields() ? Option.empty()
- : Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(new TypedProperties(config.getProps())));
- } catch (IOException e) {
- throw new HoodieIOException("KeyGenerator instantiation failed ", e);
- }
- }
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/factory/HoodieSparkKeyGeneratorFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/factory/HoodieSparkKeyGeneratorFactory.java
index dcc2eaec9eb..c655bf62543 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/factory/HoodieSparkKeyGeneratorFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/factory/HoodieSparkKeyGeneratorFactory.java
@@ -23,13 +23,15 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieKeyGeneratorException;
+import org.apache.hudi.keygen.AutoRecordGenWrapperKeyGenerator;
+import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.keygen.BuiltinKeyGenerator;
import org.apache.hudi.keygen.ComplexKeyGenerator;
import org.apache.hudi.keygen.CustomKeyGenerator;
import org.apache.hudi.keygen.GlobalDeleteKeyGenerator;
-import org.apache.hudi.keygen.AutoRecordGenWrapperKeyGenerator;
import org.apache.hudi.keygen.KeyGenUtils;
import org.apache.hudi.keygen.KeyGenerator;
import org.apache.hudi.keygen.NonpartitionedKeyGenerator;
@@ -41,14 +43,13 @@ import org.apache.hudi.keygen.constant.KeyGeneratorType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
-import static org.apache.hudi.config.HoodieWriteConfig.SPARK_SQL_MERGE_INTO_PREPPED_KEY;
import static org.apache.hudi.config.HoodieWriteConfig.KEYGENERATOR_TYPE;
+import static org.apache.hudi.config.HoodieWriteConfig.SPARK_SQL_MERGE_INTO_PREPPED_KEY;
import static org.apache.hudi.keygen.KeyGenUtils.inferKeyGeneratorType;
/**
@@ -77,26 +78,40 @@ public class HoodieSparkKeyGeneratorFactory {
"org.apache.hudi.keygen.TimestampBasedKeyGenerator");
}
- public static KeyGenerator createKeyGenerator(TypedProperties props) throws IOException {
+ public static KeyGenerator createKeyGenerator(TypedProperties props) {
String keyGeneratorClass = getKeyGeneratorClassName(props);
return createKeyGenerator(keyGeneratorClass, props);
}
- public static KeyGenerator createKeyGenerator(String keyGeneratorClass, TypedProperties props) throws IOException {
+ public static KeyGenerator createKeyGenerator(String keyGeneratorClass, TypedProperties props) {
boolean autoRecordKeyGen = KeyGenUtils.isAutoGeneratedRecordKeysEnabled(props)
//Need to prevent overwriting the keygen for spark sql merge into because we need to extract
//the recordkey from the meta cols if it exists. Sql keygen will use pkless keygen if needed.
&& !props.getBoolean(SPARK_SQL_MERGE_INTO_PREPPED_KEY, false);
- try {
- KeyGenerator keyGenerator = (KeyGenerator) ReflectionUtils.loadClass(keyGeneratorClass, props);
- if (autoRecordKeyGen) {
- return new AutoRecordGenWrapperKeyGenerator(props, (BuiltinKeyGenerator) keyGenerator);
- } else {
- // if user comes with their own key generator.
- return keyGenerator;
+ KeyGenerator keyGenerator = (KeyGenerator) ReflectionUtils.loadClass(keyGeneratorClass, props);
+ if (autoRecordKeyGen) {
+ return new AutoRecordGenWrapperKeyGenerator(props, (BuiltinKeyGenerator) keyGenerator);
+ } else {
+ // if user comes with their own key generator.
+ return keyGenerator;
+ }
+ }
+
+ /**
+ * Creates BaseKeyGenerator if meta columns are disabled.
+ *
+ * @throws HoodieException if unable instantiate or cast class to {@link BaseKeyGenerator}.
+ */
+ public static Option<BaseKeyGenerator> createBaseKeyGenerator(HoodieWriteConfig writeConfig) {
+ if (!writeConfig.populateMetaFields()) {
+ try {
+ TypedProperties typedProperties = new TypedProperties(writeConfig.getProps());
+ return Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(typedProperties));
+ } catch (ClassCastException cce) {
+ throw new HoodieException("Only BaseKeyGenerators are supported when meta columns are disabled ", cce);
}
- } catch (Throwable e) {
- throw new IOException("Could not load key generator class " + keyGeneratorClass, e);
+ } else {
+ return Option.empty();
}
}
@@ -140,8 +155,6 @@ public class HoodieSparkKeyGeneratorFactory {
return Option.of((BuiltinKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(typedProperties));
} catch (ClassCastException cce) {
throw new HoodieIOException("Only those key generators implementing BuiltInKeyGenerator interface is supported with virtual keys");
- } catch (IOException e) {
- throw new HoodieIOException("Key generator instantiation failed ", e);
}
}
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkCopyOnWriteTable.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkCopyOnWriteTable.java
index eeadd40d99e..441ac9eb1ec 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkCopyOnWriteTable.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkCopyOnWriteTable.java
@@ -31,7 +31,6 @@ import org.apache.hudi.avro.model.HoodieRollbackPlan;
import org.apache.hudi.avro.model.HoodieSavepointMetadata;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
-import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.data.HoodieData;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieBaseFile;
@@ -43,7 +42,6 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.exception.HoodieNotSupportedException;
import org.apache.hudi.io.HoodieCreateHandle;
@@ -240,15 +238,7 @@ public class HoodieSparkCopyOnWriteTable<T>
protected HoodieMergeHandle getUpdateHandle(String instantTime, String partitionPath, String fileId,
Map<String, HoodieRecord<T>> keyToNewRecords, HoodieBaseFile dataFileToBeMerged) {
- Option<BaseKeyGenerator> keyGeneratorOpt = Option.empty();
- if (!config.populateMetaFields()) {
- try {
- keyGeneratorOpt = Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(new TypedProperties(config.getProps())));
- } catch (IOException e) {
- throw new HoodieIOException("Only BaseKeyGenerator (or any key generator that extends from BaseKeyGenerator) are supported when meta "
- + "columns are disabled. Please choose the right key generator if you wish to disable meta fields.", e);
- }
- }
+ Option<BaseKeyGenerator> keyGeneratorOpt = HoodieSparkKeyGeneratorFactory.createBaseKeyGenerator(config);
return HoodieMergeHandleFactory.create(config, instantTime, this, keyToNewRecords, partitionPath, fileId,
dataFileToBeMerged, taskContextSupplier, keyGeneratorOpt);
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/SparkBootstrapCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/SparkBootstrapCommitActionExecutor.java
index 6f94139b4b7..994d66e3324 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/SparkBootstrapCommitActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/SparkBootstrapCommitActionExecutor.java
@@ -50,7 +50,6 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.data.HoodieJavaRDD;
import org.apache.hudi.exception.HoodieCommitException;
import org.apache.hudi.exception.HoodieIOException;
-import org.apache.hudi.exception.HoodieKeyGeneratorException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.keygen.KeyGeneratorInterface;
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
@@ -336,13 +335,7 @@ public class SparkBootstrapCommitActionExecutor<T>
TypedProperties properties = new TypedProperties();
properties.putAll(config.getProps());
- KeyGeneratorInterface keyGenerator;
- try {
- keyGenerator = HoodieSparkKeyGeneratorFactory.createKeyGenerator(properties);
- } catch (IOException e) {
- throw new HoodieKeyGeneratorException("Init keyGenerator failed ", e);
- }
-
+ KeyGeneratorInterface keyGenerator = HoodieSparkKeyGeneratorFactory.createKeyGenerator(properties);
BootstrapPartitionPathTranslator translator = ReflectionUtils.loadClass(config.getBootstrapPartitionPathTranslatorClass());
List<Pair<String, Pair<String, HoodieFileStatus>>> bootstrapPaths = partitions.stream()
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
index 264e00c53f9..30e3cb533b1 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
@@ -41,7 +41,6 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.data.HoodieJavaPairRDD;
import org.apache.hudi.data.HoodieJavaRDD;
import org.apache.hudi.exception.HoodieCommitException;
-import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieUpsertException;
import org.apache.hudi.execution.SparkLazyInsertIterable;
import org.apache.hudi.index.HoodieIndex;
@@ -103,13 +102,7 @@ public abstract class BaseSparkCommitActionExecutor<T> extends
WriteOperationType operationType,
Option<Map<String, String>> extraMetadata) {
super(context, config, table, instantTime, operationType, extraMetadata);
- try {
- keyGeneratorOpt = config.populateMetaFields()
- ? Option.empty()
- : Option.of((BaseKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(this.config.getProps()));
- } catch (IOException e) {
- throw new HoodieIOException("Only BaseKeyGenerators are supported when meta columns are disabled ", e);
- }
+ keyGeneratorOpt = HoodieSparkKeyGeneratorFactory.createBaseKeyGenerator(config);
}
private HoodieData<HoodieRecord<T>> clusteringHandleUpdate(HoodieData<HoodieRecord<T>> inputRecords) {
diff --git a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/bootstrap/SparkFullBootstrapDataProviderBase.java b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/bootstrap/SparkFullBootstrapDataProviderBase.java
index 6117cdcae1e..c857b61e0a4 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/bootstrap/SparkFullBootstrapDataProviderBase.java
+++ b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/bootstrap/SparkFullBootstrapDataProviderBase.java
@@ -18,19 +18,18 @@
package org.apache.hudi.bootstrap;
-import org.apache.avro.generic.GenericRecord;
import org.apache.hudi.DataSourceUtils;
import org.apache.hudi.HoodieSparkUtils;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.avro.model.HoodieFileStatus;
import org.apache.hudi.client.bootstrap.FullRecordBootstrapDataProvider;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
-import org.apache.hudi.common.model.HoodieSparkRecord;
import org.apache.hudi.common.bootstrap.FileStatusUtils;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType;
+import org.apache.hudi.common.model.HoodieSparkRecord;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
@@ -39,6 +38,8 @@ import org.apache.hudi.keygen.KeyGenerator;
import org.apache.hudi.keygen.SparkKeyGeneratorInterface;
import org.apache.hudi.keygen.constant.KeyGeneratorOptions;
import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory;
+
+import org.apache.avro.generic.GenericRecord;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.rdd.RDD;
import org.apache.spark.sql.Dataset;
@@ -69,41 +70,36 @@ public abstract class SparkFullBootstrapDataProviderBase extends FullRecordBoots
// More details at https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery
HoodieRecordType recordType = config.getRecordMerger().getRecordType();
Dataset inputDataset = sparkSession.read().format(getFormat()).option("basePath", sourceBasePath).load(filePaths);
- try {
- KeyGenerator keyGenerator = HoodieSparkKeyGeneratorFactory.createKeyGenerator(props);
- String precombineKey = props.getString("hoodie.datasource.write.precombine.field");
- String structName = tableName + "_record";
- String namespace = "hoodie." + tableName;
- if (recordType == HoodieRecordType.AVRO) {
- RDD<GenericRecord> genericRecords = HoodieSparkUtils.createRdd(inputDataset, structName, namespace, false,
- Option.empty());
- return genericRecords.toJavaRDD().map(gr -> {
- String orderingVal = HoodieAvroUtils.getNestedFieldValAsString(
- gr, precombineKey, false, props.getBoolean(
- KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key(),
- Boolean.parseBoolean(KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.defaultValue())));
- try {
- return DataSourceUtils.createHoodieRecord(gr, orderingVal, keyGenerator.getKey(gr),
- props.getString("hoodie.datasource.write.payload.class"), scala.Option.apply(null));
- } catch (IOException ioe) {
- throw new HoodieIOException(ioe.getMessage(), ioe);
- }
- });
- } else if (recordType == HoodieRecordType.SPARK) {
- SparkKeyGeneratorInterface sparkKeyGenerator = (SparkKeyGeneratorInterface) keyGenerator;
- StructType structType = inputDataset.schema();
- return inputDataset.queryExecution().toRdd().toJavaRDD().map(internalRow -> {
- String recordKey = sparkKeyGenerator.getRecordKey(internalRow, structType).toString();
- String partitionPath = sparkKeyGenerator.getPartitionPath(internalRow, structType).toString();
- HoodieKey key = new HoodieKey(recordKey, partitionPath);
- return new HoodieSparkRecord(key, internalRow, structType, false);
- });
- } else {
- throw new UnsupportedOperationException(recordType.name());
- }
-
- } catch (IOException ioe) {
- throw new HoodieIOException(ioe.getMessage(), ioe);
+ KeyGenerator keyGenerator = HoodieSparkKeyGeneratorFactory.createKeyGenerator(props);
+ String precombineKey = props.getString("hoodie.datasource.write.precombine.field");
+ String structName = tableName + "_record";
+ String namespace = "hoodie." + tableName;
+ if (recordType == HoodieRecordType.AVRO) {
+ RDD<GenericRecord> genericRecords = HoodieSparkUtils.createRdd(inputDataset, structName, namespace, false,
+ Option.empty());
+ return genericRecords.toJavaRDD().map(gr -> {
+ String orderingVal = HoodieAvroUtils.getNestedFieldValAsString(
+ gr, precombineKey, false, props.getBoolean(
+ KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key(),
+ Boolean.parseBoolean(KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.defaultValue())));
+ try {
+ return DataSourceUtils.createHoodieRecord(gr, orderingVal, keyGenerator.getKey(gr),
+ props.getString("hoodie.datasource.write.payload.class"), scala.Option.apply(null));
+ } catch (IOException ioe) {
+ throw new HoodieIOException(ioe.getMessage(), ioe);
+ }
+ });
+ } else if (recordType == HoodieRecordType.SPARK) {
+ SparkKeyGeneratorInterface sparkKeyGenerator = (SparkKeyGeneratorInterface) keyGenerator;
+ StructType structType = inputDataset.schema();
+ return inputDataset.queryExecution().toRdd().toJavaRDD().map(internalRow -> {
+ String recordKey = sparkKeyGenerator.getRecordKey(internalRow, structType).toString();
+ String partitionPath = sparkKeyGenerator.getPartitionPath(internalRow, structType).toString();
+ HoodieKey key = new HoodieKey(recordKey, partitionPath);
+ return new HoodieSparkRecord(key, internalRow, structType, false);
+ });
+ } else {
+ throw new UnsupportedOperationException(recordType.name());
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java
index 0ba8d1425e7..46e8b9f441d 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java
@@ -377,6 +377,6 @@ public class TestCustomKeyGenerator extends KeyGeneratorTestUtilities {
private static Throwable getNestedConstructorErrorCause(Exception e) {
// custom key generator will fail in the constructor, and we must unwrap the cause for asserting error messages
- return e.getCause().getCause().getCause();
+ return e.getCause().getCause();
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java
index 3cc30e86399..e7c9c723721 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java
@@ -20,6 +20,7 @@ package org.apache.hudi.keygen.factory;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieKeyGeneratorException;
import org.apache.hudi.keygen.ComplexKeyGenerator;
import org.apache.hudi.keygen.CustomKeyGenerator;
@@ -32,8 +33,6 @@ import org.apache.hudi.keygen.constant.KeyGeneratorType;
import org.junit.jupiter.api.Test;
-import java.io.IOException;
-
import static org.apache.hudi.config.HoodieWriteConfig.KEYGENERATOR_TYPE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -67,7 +66,7 @@ public class TestHoodieSparkKeyGeneratorFactory {
}
@Test
- public void testKeyGeneratorFactory() throws IOException {
+ public void testKeyGeneratorFactory() {
TypedProperties props = getCommonProps();
// set KeyGenerator type only
@@ -91,7 +90,7 @@ public class TestHoodieSparkKeyGeneratorFactory {
// set wrong class name
final TypedProperties props2 = getCommonProps();
props2.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), TestHoodieSparkKeyGeneratorFactory.class.getName());
- assertThrows(IOException.class, () -> HoodieSparkKeyGeneratorFactory.createKeyGenerator(props2));
+ assertThrows(HoodieException.class, () -> HoodieSparkKeyGeneratorFactory.createKeyGenerator(props2));
// set wrong keyGenerator type
final TypedProperties props3 = getCommonProps();
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
index 0767d055915..120304c1219 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
@@ -412,7 +412,7 @@ def testBulkInsertForDropPartitionColumn(): Unit = {
val df = spark.createDataFrame(sc.parallelize(recordsSeq), structType)
// try write to Hudi
- assertThrows[IOException] {
+ assertThrows[HoodieException] {
HoodieSparkSqlWriter.write(sqlContext, SaveMode.Append, tableOpts - DataSourceWriteOptions.PARTITIONPATH_FIELD.key, df)
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
index dd613ce1153..f710786e41f 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
@@ -1104,7 +1104,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
writer.save(basePath)
fail("should fail when invalid PartitionKeyType is provided!")
} catch {
- case e: Exception => assertTrue(e.getCause.getMessage.contains("Unable to instantiate class org.apache.hudi.keygen.CustomKeyGenerator"))
+ case e: Exception => assertTrue(e.getMessage.contains("Unable to instantiate class org.apache.hudi.keygen.CustomKeyGenerator"))
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithCustomKeyGenerator.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithCustomKeyGenerator.scala
index c85eb40bca7..0fc2976c7fe 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithCustomKeyGenerator.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithCustomKeyGenerator.scala
@@ -33,8 +33,6 @@ import org.joda.time.format.DateTimeFormat
import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
import org.slf4j.LoggerFactory
-import java.io.IOException
-
/**
* Tests Spark SQL DML with custom key generator and write configs.
*/
@@ -288,7 +286,7 @@ class TestSparkSqlWithCustomKeyGenerator extends HoodieSparkSqlTestBase {
// INSERT INTO should fail for tableNameCustom1
val sourceTableName = tableNameCustom1 + "_source"
prepareParquetSource(sourceTableName, Seq("(7, 'a7', 1399.0, 1706800227, 'cat1')"))
- assertThrows[IOException] {
+ assertThrows[HoodieException] {
spark.sql(
s"""
| INSERT INTO $tableNameCustom1
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
index bc6332c842d..14aa3b5d2e9 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
@@ -390,7 +390,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
@Test
public void testPropsWithInvalidKeyGenerator() {
- Exception e = assertThrows(IOException.class, () -> {
+ Exception e = assertThrows(HoodieException.class, () -> {
String tableBasePath = basePath + "/test_table_invalid_key_gen";
HoodieDeltaStreamer deltaStreamer =
new HoodieDeltaStreamer(TestHelpers.makeConfig(tableBasePath, WriteOperationType.BULK_INSERT,
@@ -399,7 +399,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
}, "Should error out when setting the key generator class property to an invalid value");
// expected
LOG.warn("Expected error during getting the key generator", e);
- assertTrue(e.getMessage().contains("Could not load key generator class invalid"));
+ assertTrue(e.getMessage().contains("Unable to load class"));
}
private static Stream<Arguments> provideInferKeyGenArgs() {
(hudi) 07/16: [HUDI-7631] Clean up usage of CachingPath outside hudi-common module (#11059)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 4180f7827fb90558dd5808c3c82d3ad1c6180281
Author: Vova Kolmakov <wo...@gmail.com>
AuthorDate: Sun Apr 21 11:58:48 2024 +0700
[HUDI-7631] Clean up usage of CachingPath outside hudi-common module (#11059)
---
.../metadata/HoodieBackedTableMetadataWriter.java | 57 ++++++++--------------
.../hudi/client/utils/SparkPartitionUtils.java | 4 +-
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 28 +++++------
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 20 ++++----
.../java/org/apache/hudi/common/fs/FSUtils.java | 4 ++
.../view/HoodieTablePreCommitFileSystemView.java | 4 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 12 ++---
.../apache/hudi/SparkHoodieTableFileIndex.scala | 2 +-
.../utilities/HoodieMetadataTableValidator.java | 5 +-
.../utilities/streamer/SparkSampleWritesUtils.java | 15 +++---
10 files changed, 65 insertions(+), 86 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 0714f27d0e8..8970640c6ee 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -67,18 +67,12 @@ import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieIndexException;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.exception.TableNotFoundException;
-import org.apache.hudi.hadoop.fs.CachingPath;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
-import org.apache.hudi.hadoop.fs.SerializablePath;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.table.BulkInsertPartitioner;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -593,15 +587,14 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
* @return List consisting of {@code DirectoryInfo} for each partition found.
*/
private List<DirectoryInfo> listAllPartitionsFromFilesystem(String initializationTime) {
- List<SerializablePath> pathsToList = new LinkedList<>();
- pathsToList.add(new SerializablePath(new CachingPath(dataWriteConfig.getBasePath())));
+ List<StoragePath> pathsToList = new LinkedList<>();
+ pathsToList.add(new StoragePath(dataWriteConfig.getBasePath()));
List<DirectoryInfo> partitionsToBootstrap = new LinkedList<>();
final int fileListingParallelism = metadataWriteConfig.getFileListingParallelism();
- SerializableConfiguration conf = new SerializableConfiguration(dataMetaClient.getHadoopConf());
final String dirFilterRegex = dataWriteConfig.getMetadataConfig().getDirectoryFilterRegex();
final String datasetBasePath = dataMetaClient.getBasePathV2().toString();
- SerializablePath serializableBasePath = new SerializablePath(new CachingPath(datasetBasePath));
+ StoragePath storageBasePath = new StoragePath(datasetBasePath);
while (!pathsToList.isEmpty()) {
// In each round we will list a section of directories
@@ -609,9 +602,8 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
// List all directories in parallel
engineContext.setJobStatus(this.getClass().getSimpleName(), "Listing " + numDirsToList + " partitions from filesystem");
List<DirectoryInfo> processedDirectories = engineContext.map(pathsToList.subList(0, numDirsToList), path -> {
- FileSystem fs = path.get().getFileSystem(conf.get());
- String relativeDirPath = FSUtils.getRelativePartitionPath(serializableBasePath.get(), path.get());
- return new DirectoryInfo(relativeDirPath, fs.listStatus(path.get()), initializationTime);
+ String relativeDirPath = FSUtils.getRelativePartitionPath(storageBasePath, path);
+ return new DirectoryInfo(relativeDirPath, metadataMetaClient.getStorage().listDirectEntries(path), initializationTime);
}, numDirsToList);
pathsToList = new LinkedList<>(pathsToList.subList(numDirsToList, pathsToList.size()));
@@ -632,9 +624,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
partitionsToBootstrap.add(dirInfo);
} else {
// Add sub-dirs to the queue
- pathsToList.addAll(dirInfo.getSubDirectories().stream()
- .map(path -> new SerializablePath(new CachingPath(path.toUri())))
- .collect(Collectors.toList()));
+ pathsToList.addAll(dirInfo.getSubDirectories());
}
}
}
@@ -651,14 +641,9 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
private List<DirectoryInfo> listAllPartitionsFromMDT(String initializationTime) throws IOException {
List<DirectoryInfo> dirinfoList = new LinkedList<>();
List<String> allPartitionPaths = metadata.getAllPartitionPaths().stream()
- .map(partitionPath -> dataWriteConfig.getBasePath() + "/" + partitionPath).collect(Collectors.toList());
- Map<String, FileStatus[]> partitionFileMap = metadata.getAllFilesInPartitions(allPartitionPaths)
- .entrySet()
- .stream()
- .collect(Collectors.toMap(e -> e.getKey(),
- e -> e.getValue().stream().map(status -> HadoopFSUtils.convertToHadoopFileStatus(status))
- .toArray(FileStatus[]::new)));
- for (Map.Entry<String, FileStatus[]> entry : partitionFileMap.entrySet()) {
+ .map(partitionPath -> dataWriteConfig.getBasePath() + StoragePath.SEPARATOR_CHAR + partitionPath).collect(Collectors.toList());
+ Map<String, List<StoragePathInfo>> partitionFileMap = metadata.getAllFilesInPartitions(allPartitionPaths);
+ for (Map.Entry<String, List<StoragePathInfo>> entry : partitionFileMap.entrySet()) {
dirinfoList.add(new DirectoryInfo(entry.getKey(), entry.getValue(), initializationTime));
}
return dirinfoList;
@@ -1495,31 +1480,31 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
// Map of filenames within this partition to their respective sizes
private final HashMap<String, Long> filenameToSizeMap;
// List of directories within this partition
- private final List<Path> subDirectories = new ArrayList<>();
+ private final List<StoragePath> subDirectories = new ArrayList<>();
// Is this a hoodie partition
private boolean isHoodiePartition = false;
- public DirectoryInfo(String relativePath, FileStatus[] fileStatus, String maxInstantTime) {
+ public DirectoryInfo(String relativePath, List<StoragePathInfo> pathInfos, String maxInstantTime) {
this.relativePath = relativePath;
// Pre-allocate with the maximum length possible
- filenameToSizeMap = new HashMap<>(fileStatus.length);
+ filenameToSizeMap = new HashMap<>(pathInfos.size());
- for (FileStatus status : fileStatus) {
- if (status.isDirectory()) {
+ for (StoragePathInfo pathInfo : pathInfos) {
+ if (pathInfo.isDirectory()) {
// Ignore .hoodie directory as there cannot be any partitions inside it
- if (!status.getPath().getName().equals(HoodieTableMetaClient.METAFOLDER_NAME)) {
- this.subDirectories.add(status.getPath());
+ if (!pathInfo.getPath().getName().equals(HoodieTableMetaClient.METAFOLDER_NAME)) {
+ this.subDirectories.add(pathInfo.getPath());
}
- } else if (status.getPath().getName().startsWith(HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE_PREFIX)) {
+ } else if (pathInfo.getPath().getName().startsWith(HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE_PREFIX)) {
// Presence of partition meta file implies this is a HUDI partition
this.isHoodiePartition = true;
- } else if (FSUtils.isDataFile(status.getPath())) {
+ } else if (FSUtils.isDataFile(pathInfo.getPath())) {
// Regular HUDI data file (base file or log file)
- String dataFileCommitTime = FSUtils.getCommitTime(status.getPath().getName());
+ String dataFileCommitTime = FSUtils.getCommitTime(pathInfo.getPath().getName());
// Limit the file listings to files which were created before the maxInstant time.
if (HoodieTimeline.compareTimestamps(dataFileCommitTime, LESSER_THAN_OR_EQUALS, maxInstantTime)) {
- filenameToSizeMap.put(status.getPath().getName(), status.getLen());
+ filenameToSizeMap.put(pathInfo.getPath().getName(), pathInfo.getLength());
}
}
}
@@ -1537,7 +1522,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
return isHoodiePartition;
}
- List<Path> getSubDirectories() {
+ List<StoragePath> getSubDirectories() {
return subDirectories;
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkPartitionUtils.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkPartitionUtils.java
index d6545f247b6..e8db1b3515d 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkPartitionUtils.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkPartitionUtils.java
@@ -22,7 +22,7 @@ import org.apache.hudi.AvroConversionUtils;
import org.apache.hudi.HoodieSparkUtils;
import org.apache.hudi.SparkAdapterSupport$;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.hadoop.fs.CachingPath;
+import org.apache.hudi.storage.StoragePath;
import org.apache.avro.Schema;
import org.apache.hadoop.conf.Configuration;
@@ -43,7 +43,7 @@ public class SparkPartitionUtils {
return HoodieSparkUtils.parsePartitionColumnValues(
partitionFields.get(),
partitionPath,
- new CachingPath(basePath),
+ new StoragePath(basePath),
AvroConversionUtils.convertAvroSchemaToStructType(writerSchema),
hadoopConf.get("timeZone", SQLConf.get().sessionLocalTimeZone()),
sparkParsePartitionUtil,
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
index 0d164f379fe..2a8c395d0d5 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
@@ -34,14 +34,11 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieInsertException;
-import org.apache.hudi.hadoop.fs.CachingPath;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.marker.WriteMarkersFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.unsafe.types.UTF8String;
@@ -67,7 +64,7 @@ public class HoodieRowCreateHandle implements Serializable {
private final HoodieWriteConfig writeConfig;
private final String partitionPath;
- private final Path path;
+ private final StoragePath path;
private final String fileId;
private final boolean populateMetaFields;
@@ -116,12 +113,11 @@ public class HoodieRowCreateHandle implements Serializable {
this.currTimer = HoodieTimer.start();
HoodieStorage storage = table.getMetaClient().getStorage();
- FileSystem fs = (FileSystem) storage.getFileSystem();
String writeToken = getWriteToken(taskPartitionId, taskId, taskEpochId);
String fileName = FSUtils.makeBaseFileName(instantTime, writeToken, this.fileId,
table.getBaseFileExtension());
- this.path = makeNewPath(fs, partitionPath, fileName, writeConfig);
+ this.path = makeNewPath(storage, partitionPath, fileName, writeConfig);
this.populateMetaFields = writeConfig.populateMetaFields();
this.fileName = UTF8String.fromString(path.getName());
@@ -147,13 +143,12 @@ public class HoodieRowCreateHandle implements Serializable {
createMarkerFile(partitionPath, fileName, instantTime, table, writeConfig);
- this.fileWriter = HoodieInternalRowFileWriterFactory.getInternalRowFileWriter(
- new StoragePath(path.toUri()), table, writeConfig, structType);
+ this.fileWriter = HoodieInternalRowFileWriterFactory.getInternalRowFileWriter(path, table, writeConfig, structType);
} catch (IOException e) {
throw new HoodieInsertException("Failed to initialize file writer for path " + path, e);
}
- LOG.info("New handle created for partition: " + partitionPath + " with fileId " + fileId);
+ LOG.info("New handle created for partition: {} with fileId {}", partitionPath, fileId);
}
/**
@@ -242,9 +237,8 @@ public class HoodieRowCreateHandle implements Serializable {
stat.setNumInserts(writeStatus.getTotalRecords());
stat.setPrevCommit(HoodieWriteStat.NULL_COMMIT);
stat.setFileId(fileId);
- stat.setPath(new StoragePath(writeConfig.getBasePath()), new StoragePath(path.toUri()));
- long fileSizeInBytes = FSUtils.getFileSize(table.getMetaClient().getStorage(),
- new StoragePath(path.toUri()));
+ stat.setPath(new StoragePath(writeConfig.getBasePath()), path);
+ long fileSizeInBytes = FSUtils.getFileSize(table.getMetaClient().getStorage(), path);
stat.setTotalWriteBytes(fileSizeInBytes);
stat.setFileSizeInBytes(fileSizeInBytes);
stat.setTotalWriteErrors(writeStatus.getTotalErrorRecords());
@@ -261,16 +255,16 @@ public class HoodieRowCreateHandle implements Serializable {
return path.getName();
}
- private static Path makeNewPath(FileSystem fs, String partitionPath, String fileName, HoodieWriteConfig writeConfig) {
- Path path = FSUtils.constructAbsolutePathInHadoopPath(writeConfig.getBasePath(), partitionPath);
+ private static StoragePath makeNewPath(HoodieStorage storage, String partitionPath, String fileName, HoodieWriteConfig writeConfig) {
+ StoragePath path = new StoragePath(writeConfig.getBasePath(), partitionPath);
try {
- if (!fs.exists(path)) {
- fs.mkdirs(path); // create a new partition as needed.
+ if (!storage.exists(path)) {
+ storage.createDirectory(path); // create a new partition as needed.
}
} catch (IOException e) {
throw new HoodieIOException("Failed to make dir " + path, e);
}
- return new CachingPath(path.toString(), fileName);
+ return new StoragePath(path, fileName);
}
/**
diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala
index 3393da6bd83..7febf2a2ced 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala
@@ -25,7 +25,7 @@ import org.apache.hudi.HoodieConversionUtils.toScalaOption
import org.apache.hudi.avro.{AvroSchemaUtils, HoodieAvroUtils}
import org.apache.hudi.client.utils.SparkRowSerDe
import org.apache.hudi.common.model.HoodieRecord
-import org.apache.hudi.hadoop.fs.CachingPath
+import org.apache.hudi.storage.StoragePath
import org.apache.hudi.util.ExceptionWrappingIterator
import org.apache.spark.SPARK_VERSION
import org.apache.spark.internal.Logging
@@ -237,7 +237,7 @@ object HoodieSparkUtils extends SparkAdapterSupport with SparkVersionsSupport wi
def parsePartitionColumnValues(partitionColumns: Array[String],
partitionPath: String,
- basePath: Path,
+ basePath: StoragePath,
schema: StructType,
timeZoneId: String,
sparkParsePartitionUtil: SparkParsePartitionUtil,
@@ -246,7 +246,7 @@ object HoodieSparkUtils extends SparkAdapterSupport with SparkVersionsSupport wi
// This is a non-partitioned table
Array.empty
} else {
- val partitionFragments = partitionPath.split("/")
+ val partitionFragments = partitionPath.split(StoragePath.SEPARATOR)
if (partitionFragments.length != partitionColumns.length) {
if (partitionColumns.length == 1) {
// If the partition column size is not equal to the partition fragment size
@@ -290,9 +290,9 @@ object HoodieSparkUtils extends SparkAdapterSupport with SparkVersionsSupport wi
} else {
partition
}
- }.mkString("/")
+ }.mkString(StoragePath.SEPARATOR)
- val pathWithPartitionName = new CachingPath(basePath, CachingPath.createRelativePathUnsafe(partitionWithName))
+ val pathWithPartitionName = new StoragePath(basePath, partitionWithName)
val partitionSchema = StructType(schema.fields.filter(f => partitionColumns.contains(f.name)))
val partitionValues = parsePartitionPath(pathWithPartitionName, partitionSchema, timeZoneId,
sparkParsePartitionUtil, basePath, shouldValidatePartitionCols)
@@ -301,14 +301,14 @@ object HoodieSparkUtils extends SparkAdapterSupport with SparkVersionsSupport wi
}
}
- private def parsePartitionPath(partitionPath: Path, partitionSchema: StructType, timeZoneId: String,
- sparkParsePartitionUtil: SparkParsePartitionUtil, basePath: Path,
+ private def parsePartitionPath(partitionPath: StoragePath, partitionSchema: StructType, timeZoneId: String,
+ sparkParsePartitionUtil: SparkParsePartitionUtil, basePath: StoragePath,
shouldValidatePartitionCols: Boolean): Seq[Any] = {
val partitionDataTypes = partitionSchema.map(f => f.name -> f.dataType).toMap
sparkParsePartitionUtil.parsePartition(
- partitionPath,
+ new Path(partitionPath.toUri),
typeInference = false,
- Set(basePath),
+ Set(new Path(basePath.toUri)),
partitionDataTypes,
getTimeZone(timeZoneId),
validatePartitionValues = shouldValidatePartitionCols
@@ -329,7 +329,7 @@ object HoodieSparkUtils extends SparkAdapterSupport with SparkVersionsSupport wi
partitionVals(index) = fragment.substring(fragment.indexOf("=") + 1)
} else {
- partitionVals(index) += "/" + fragment
+ partitionVals(index) += StoragePath.SEPARATOR + fragment
}
}
return partitionVals
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
index 0685d8d4a88..0b6d8699631 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
@@ -567,6 +567,10 @@ public class FSUtils {
return isBaseFile(path) || isLogFile(path);
}
+ public static boolean isDataFile(StoragePath path) {
+ return isBaseFile(path) || isLogFile(path);
+ }
+
/**
* Get the names of all the base and log files in the given partition path.
*/
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTablePreCommitFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTablePreCommitFileSystemView.java
index afae30ca8e2..ea6b8f429bd 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTablePreCommitFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTablePreCommitFileSystemView.java
@@ -21,7 +21,7 @@ package org.apache.hudi.common.table.view;
import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.hudi.hadoop.fs.CachingPath;
+import org.apache.hudi.storage.StoragePath;
import java.util.Collections;
import java.util.List;
@@ -71,7 +71,7 @@ public class HoodieTablePreCommitFileSystemView {
Map<String, HoodieBaseFile> newFilesWrittenForPartition = filesWritten.stream()
.filter(file -> partitionStr.equals(file.getPartitionPath()))
.collect(Collectors.toMap(HoodieWriteStat::getFileId, writeStat ->
- new HoodieBaseFile(new CachingPath(tableMetaClient.getBasePath(), writeStat.getPath()).toString(), writeStat.getFileId(), preCommitInstantTime, null)));
+ new HoodieBaseFile(new StoragePath(tableMetaClient.getBasePath(), writeStat.getPath()).toString(), writeStat.getFileId(), preCommitInstantTime, null)));
Stream<HoodieBaseFile> committedBaseFiles = this.completedCommitsFileSystemView.getLatestBaseFiles(partitionStr);
Map<String, HoodieBaseFile> allFileIds = committedBaseFiles
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
index d4ba0f714a9..c228d3db0ed 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
@@ -39,7 +39,7 @@ import org.apache.hudi.common.util.{ConfigUtils, StringUtils}
import org.apache.hudi.config.HoodieBootstrapConfig.DATA_QUERIES_ONLY
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.exception.HoodieException
-import org.apache.hudi.hadoop.fs.{CachingPath, HadoopFSUtils}
+import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.internal.schema.InternalSchema
import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter
import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper}
@@ -68,8 +68,6 @@ import org.apache.spark.sql.sources.{BaseRelation, Filter, PrunedFilteredScan}
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.{Row, SQLContext, SparkSession}
-import java.net.URI
-
import scala.collection.JavaConverters._
import scala.util.{Failure, Success, Try}
@@ -489,14 +487,14 @@ abstract class HoodieBaseRelation(val sqlContext: SQLContext,
protected def getPartitionColumnsAsInternalRowInternal(file: StoragePathInfo, basePath: Path,
extractPartitionValuesFromPartitionPath: Boolean): InternalRow = {
if (extractPartitionValuesFromPartitionPath) {
- val tablePathWithoutScheme = CachingPath.getPathWithoutSchemeAndAuthority(basePath)
- val partitionPathWithoutScheme = CachingPath.getPathWithoutSchemeAndAuthority(new Path(file.getPath.getParent.toUri))
- val relativePath = new URI(tablePathWithoutScheme.toString).relativize(new URI(partitionPathWithoutScheme.toString)).toString
+ val tablePathWithoutScheme = new StoragePath(basePath.toUri).getPathWithoutSchemeAndAuthority
+ val partitionPathWithoutScheme = new StoragePath(file.getPath.getParent.toUri).getPathWithoutSchemeAndAuthority
+ val relativePath = tablePathWithoutScheme.toUri.relativize(partitionPathWithoutScheme.toUri).toString
val timeZoneId = conf.get("timeZone", sparkSession.sessionState.conf.sessionLocalTimeZone)
val rowValues = HoodieSparkUtils.parsePartitionColumnValues(
partitionColumns,
relativePath,
- basePath,
+ new StoragePath(basePath.toUri),
tableStructSchema,
timeZoneId,
sparkAdapter.getSparkParsePartitionUtil,
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkHoodieTableFileIndex.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkHoodieTableFileIndex.scala
index 5dabebefd7f..9655f2ae4e0 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkHoodieTableFileIndex.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkHoodieTableFileIndex.scala
@@ -401,7 +401,7 @@ class SparkHoodieTableFileIndex(spark: SparkSession,
}
protected def doParsePartitionColumnValues(partitionColumns: Array[String], partitionPath: String): Array[Object] = {
- HoodieSparkUtils.parsePartitionColumnValues(partitionColumns, partitionPath, new Path(getBasePath.toUri), schema,
+ HoodieSparkUtils.parsePartitionColumnValues(partitionColumns, partitionPath, getBasePath, schema,
configProperties.getString(DateTimeUtils.TIMEZONE_OPTION, SQLConf.get.sessionLocalTimeZone),
sparkParsePartitionUtil, shouldValidatePartitionColumns(spark))
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
index 8a2ded37fd5..6265f0ba3db 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
@@ -103,7 +103,6 @@ import java.util.stream.Collectors;
import scala.Tuple2;
import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
-import static org.apache.hudi.hadoop.fs.CachingPath.getPathWithoutSchemeAndAuthority;
import static org.apache.hudi.metadata.HoodieTableMetadata.getMetadataTableBasePath;
/**
@@ -1244,8 +1243,8 @@ public class HoodieMetadataTableValidator implements Serializable {
}
private String getRelativePath(String basePath, String absoluteFilePath) {
- String basePathStr = getPathWithoutSchemeAndAuthority(new Path(basePath)).toString();
- String absoluteFilePathStr = getPathWithoutSchemeAndAuthority(new Path(absoluteFilePath)).toString();
+ String basePathStr = new StoragePath(basePath).getPathWithoutSchemeAndAuthority().toString();
+ String absoluteFilePathStr = new StoragePath(absoluteFilePath).getPathWithoutSchemeAndAuthority().toString();
if (!absoluteFilePathStr.startsWith(basePathStr)) {
throw new IllegalArgumentException("File path does not belong to the base path! basePath="
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SparkSampleWritesUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SparkSampleWritesUtils.java
index e7dca04bbe7..01c2ab7ef11 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SparkSampleWritesUtils.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SparkSampleWritesUtils.java
@@ -32,12 +32,11 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.hadoop.fs.CachingPath;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.slf4j.Logger;
@@ -80,7 +79,7 @@ public class SparkSampleWritesUtils {
Pair<Boolean, String> result = doSampleWrites(jsc, recordsOpt, writeConfig, instantTime);
if (result.getLeft()) {
long avgSize = getAvgSizeFromSampleWrites(jsc, result.getRight());
- LOG.info("Overwriting record size estimate to " + avgSize);
+ LOG.info("Overwriting record size estimate to {}", avgSize);
TypedProperties props = writeConfig.getProps();
props.put(COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(), String.valueOf(avgSize));
return Option.of(HoodieWriteConfig.newBuilder().withProperties(props).build());
@@ -121,7 +120,7 @@ public class SparkSampleWritesUtils {
sampleWriteClient.startCommitWithTime(instantTime);
JavaRDD<WriteStatus> writeStatusRDD = sampleWriteClient.bulkInsert(jsc.parallelize(samples, 1), instantTime);
if (writeStatusRDD.filter(WriteStatus::hasErrors).count() > 0) {
- LOG.error(String.format("sample writes for table %s failed with errors.", writeConfig.getTableName()));
+ LOG.error("sample writes for table {} failed with errors.", writeConfig.getTableName());
if (LOG.isTraceEnabled()) {
LOG.trace("Printing out the top 100 errors");
writeStatusRDD.filter(WriteStatus::hasErrors).take(100).forEach(ws -> {
@@ -139,10 +138,10 @@ public class SparkSampleWritesUtils {
}
private static String getSampleWritesBasePath(JavaSparkContext jsc, HoodieWriteConfig writeConfig, String instantTime) throws IOException {
- Path basePath = new CachingPath(writeConfig.getBasePath(), SAMPLE_WRITES_FOLDER_PATH + StoragePath.SEPARATOR + instantTime);
- FileSystem fs = HadoopFSUtils.getFs(basePath, jsc.hadoopConfiguration());
- if (fs.exists(basePath)) {
- fs.delete(basePath, true);
+ StoragePath basePath = new StoragePath(writeConfig.getBasePath(), SAMPLE_WRITES_FOLDER_PATH + StoragePath.SEPARATOR + instantTime);
+ HoodieStorage storage = getMetaClient(jsc, writeConfig.getBasePath()).getStorage();
+ if (storage.exists(basePath)) {
+ storage.deleteDirectory(basePath);
}
return basePath.toString();
}
(hudi) 11/16: [HUDI-7608] Fix Flink table creation configuration not taking effect when writing to Spark (#11005)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 278f57828fd7a26709b8446a58094de9ce6ee359
Author: empcl <15...@qq.com>
AuthorDate: Tue Apr 23 13:09:10 2024 +0800
[HUDI-7608] Fix Flink table creation configuration not taking effect when writing to Spark (#11005)
---
.../hudi/table/catalog/HoodieHiveCatalog.java | 5 +++
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 40 ++++++++++++++++++++++
2 files changed, 45 insertions(+)
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HoodieHiveCatalog.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HoodieHiveCatalog.java
index 09bf9460635..98e4cb793a0 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HoodieHiveCatalog.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HoodieHiveCatalog.java
@@ -556,6 +556,11 @@ public class HoodieHiveCatalog extends AbstractCatalog {
hiveTable.setCreateTime((int) (System.currentTimeMillis() / 1000));
Map<String, String> properties = new HashMap<>(table.getOptions());
+ if (properties.containsKey(FlinkOptions.INDEX_TYPE.key())
+ && !properties.containsKey(HoodieIndexConfig.INDEX_TYPE.key())) {
+ properties.put(HoodieIndexConfig.INDEX_TYPE.key(), properties.get(FlinkOptions.INDEX_TYPE.key()));
+ }
+ properties.remove(FlinkOptions.INDEX_TYPE.key());
hiveConf.getAllProperties().forEach((k, v) -> properties.put("hadoop." + k, String.valueOf(v)));
if (external) {
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieHiveCatalog.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieHiveCatalog.java
index 1ef03291e9a..24621e1b8d7 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieHiveCatalog.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieHiveCatalog.java
@@ -26,6 +26,7 @@ import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.exception.HoodieCatalogException;
import org.apache.hudi.keygen.ComplexAvroKeyGenerator;
@@ -258,6 +259,45 @@ public class TestHoodieHiveCatalog {
assertEquals(keyGeneratorClassName, NonpartitionedAvroKeyGenerator.class.getName());
}
+ @Test
+ void testCreateTableWithIndexType() throws TableNotExistException, TableAlreadyExistException, DatabaseNotExistException {
+ Map<String, String> options = new HashMap<>();
+ options.put(FactoryUtil.CONNECTOR.key(), "hudi");
+ // hoodie.index.type
+ options.put(HoodieIndexConfig.INDEX_TYPE.key(), "BUCKET");
+ CatalogTable table =
+ new CatalogTableImpl(schema, partitions, options, "hudi table");
+ hoodieCatalog.createTable(tablePath, table, false);
+ Map<String, String> params = hoodieCatalog.getHiveTable(tablePath).getParameters();
+ assertResult(params, "BUCKET");
+ options.remove(HoodieIndexConfig.INDEX_TYPE.key());
+
+ // index.type
+ options.put(FlinkOptions.INDEX_TYPE.key(), FlinkOptions.INDEX_TYPE.defaultValue());
+ table =
+ new CatalogTableImpl(schema, partitions, options, "hudi table");
+ ObjectPath newTablePath1 = new ObjectPath("default", "test" + System.currentTimeMillis());
+ hoodieCatalog.createTable(newTablePath1, table, false);
+
+ params = hoodieCatalog.getHiveTable(newTablePath1).getParameters();
+ assertResult(params, FlinkOptions.INDEX_TYPE.defaultValue());
+
+ // index.type + hoodie.index.type
+ options.put(HoodieIndexConfig.INDEX_TYPE.key(), "BUCKET");
+ table = new CatalogTableImpl(schema, partitions, options, "hudi table");
+ ObjectPath newTablePath2 = new ObjectPath("default", "test" + System.currentTimeMillis());
+ hoodieCatalog.createTable(newTablePath2, table, false);
+
+ params = hoodieCatalog.getHiveTable(newTablePath2).getParameters();
+ assertResult(params, "BUCKET");
+ }
+
+ private void assertResult(Map<String, String> params, String index) {
+ assertTrue(params.containsKey(HoodieIndexConfig.INDEX_TYPE.key()));
+ assertFalse(params.containsKey(FlinkOptions.INDEX_TYPE.key()));
+ assertThat(params.get(HoodieIndexConfig.INDEX_TYPE.key()), is(index));
+ }
+
@Test
void testCreateTableWithoutPreCombineKey() throws TableAlreadyExistException, DatabaseNotExistException, IOException, TableNotExistException {
String db = "default";
(hudi) 10/16: [MINOR] Reuse MetadataPartitionType enum to get all partition paths (#11068)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 7cadc96b0495ccd695682897b6b85a8f1186704b
Author: Sagar Sumit <sa...@gmail.com>
AuthorDate: Mon Apr 22 22:00:35 2024 +0530
[MINOR] Reuse MetadataPartitionType enum to get all partition paths (#11068)
---
.../org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java | 12 +-----------
.../java/org/apache/hudi/metadata/MetadataPartitionType.java | 12 ++++++++++++
2 files changed, 13 insertions(+), 11 deletions(-)
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
index 1f383cdd5d3..313101a355c 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
@@ -39,7 +39,6 @@ import org.apache.hudi.common.table.view.SyncableFileSystemView;
import org.apache.hudi.common.testutils.HoodieMetadataTestTable;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestUtils;
-import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
@@ -99,10 +98,6 @@ import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_F
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
import static org.apache.hudi.common.util.MarkerUtils.MARKERS_FILENAME_PREFIX;
import static org.apache.hudi.common.util.PartitionPathEncodeUtils.DEPRECATED_DEFAULT_PARTITION_PATH;
-import static org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_BLOOM_FILTERS;
-import static org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_COLUMN_STATS;
-import static org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_FILES;
-import static org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_RECORD_INDEX;
import static org.apache.hudi.metadata.MetadataPartitionType.RECORD_INDEX;
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -557,12 +552,7 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase {
// validate the relevant table states before downgrade
java.nio.file.Path recordIndexPartitionPath = Paths.get(basePath,
METADATA_TABLE_FOLDER_PATH, RECORD_INDEX.getPartitionPath());
- Set<String> allPartitions = CollectionUtils.createImmutableSet(
- PARTITION_NAME_FILES,
- PARTITION_NAME_COLUMN_STATS,
- PARTITION_NAME_BLOOM_FILTERS,
- PARTITION_NAME_RECORD_INDEX
- );
+ Set<String> allPartitions = MetadataPartitionType.getAllPartitionPaths();
assertTrue(Files.exists(recordIndexPartitionPath), "record index partition should exist.");
assertEquals(allPartitions, metaClient.getTableConfig().getMetadataPartitions(),
TABLE_METADATA_PARTITIONS.key() + " should contain all partitions.");
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java b/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java
index 81a6b43c4f5..ef0806d3a61 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java
@@ -18,8 +18,11 @@
package org.apache.hudi.metadata;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
/**
* Partition types for metadata table.
@@ -57,6 +60,15 @@ public enum MetadataPartitionType {
return Collections.singletonList(MetadataPartitionType.RECORD_INDEX);
}
+ /**
+ * Returns the set of all metadata partition names.
+ */
+ public static Set<String> getAllPartitionPaths() {
+ return Arrays.stream(values())
+ .map(MetadataPartitionType::getPartitionPath)
+ .collect(Collectors.toSet());
+ }
+
@Override
public String toString() {
return "Metadata partition {"
(hudi) 09/16: [HUDI-7655] Minor fix to rli validation with MDT validator (#11060)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit beb20386e0315eb0867ef769bf685bb770bcf9bf
Author: Sivabalan Narayanan <n....@gmail.com>
AuthorDate: Sun Apr 21 11:19:45 2024 -0700
[HUDI-7655] Minor fix to rli validation with MDT validator (#11060)
---
.../org/apache/hudi/utilities/HoodieMetadataTableValidator.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
index 6265f0ba3db..a5d002ccd73 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
@@ -959,6 +959,7 @@ public class HoodieMetadataTableValidator implements Serializable {
int numErrorSamples = cfg.numRecordIndexErrorSamples;
Pair<Long, List<String>> result = keyToLocationOnFsRdd.fullOuterJoin(keyToLocationFromRecordIndexRdd, cfg.recordIndexParallelism)
.map(e -> {
+ String recordKey = e._1;
Optional<Pair<String, String>> locationOnFs = e._2._1;
Optional<Pair<String, String>> locationFromRecordIndex = e._2._2;
List<String> errorSampleList = new ArrayList<>();
@@ -967,13 +968,13 @@ public class HoodieMetadataTableValidator implements Serializable {
&& locationOnFs.get().getRight().equals(locationFromRecordIndex.get().getRight())) {
return Pair.of(0L, errorSampleList);
}
- errorSampleList.add(constructLocationInfoString(locationOnFs, locationFromRecordIndex));
+ errorSampleList.add(constructLocationInfoString(recordKey, locationOnFs, locationFromRecordIndex));
return Pair.of(1L, errorSampleList);
}
if (!locationOnFs.isPresent() && !locationFromRecordIndex.isPresent()) {
return Pair.of(0L, errorSampleList);
}
- errorSampleList.add(constructLocationInfoString(locationOnFs, locationFromRecordIndex));
+ errorSampleList.add(constructLocationInfoString(recordKey, locationOnFs, locationFromRecordIndex));
return Pair.of(1L, errorSampleList);
})
.reduce((pair1, pair2) -> {
@@ -1030,9 +1031,10 @@ public class HoodieMetadataTableValidator implements Serializable {
}
}
- private String constructLocationInfoString(Optional<Pair<String, String>> locationOnFs,
+ private String constructLocationInfoString(String recordKey, Optional<Pair<String, String>> locationOnFs,
Optional<Pair<String, String>> locationFromRecordIndex) {
StringBuilder sb = new StringBuilder();
+ sb.append("Record key " + recordKey + " -> ");
sb.append("FS: ");
if (locationOnFs.isPresent()) {
sb.append(locationOnFs.get());
(hudi) 13/16: [MINOR] Fixe naming of methods in HoodieMetadataConfig (#11076)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 48890500ef66e6aad4138fdbba0e70481e136bbb
Author: Vova Kolmakov <wo...@gmail.com>
AuthorDate: Wed Apr 24 08:05:39 2024 +0700
[MINOR] Fixe naming of methods in HoodieMetadataConfig (#11076)
---
.../java/org/apache/hudi/config/HoodieWriteConfig.java | 2 +-
.../hudi/table/action/index/RunIndexActionExecutor.java | 2 +-
.../apache/hudi/testutils/HoodieJavaClientTestHarness.java | 2 +-
.../hudi/testutils/HoodieSparkClientTestHarness.java | 2 +-
.../apache/hudi/common/config/HoodieMetadataConfig.java | 14 +++++---------
.../java/org/apache/hudi/metadata/BaseTableMetadata.java | 4 ++--
.../apache/hudi/metadata/HoodieBackedTableMetadata.java | 2 +-
.../java/org/apache/hudi/metadata/HoodieTableMetadata.java | 2 +-
.../org/apache/hudi/metadata/HoodieTableMetadataUtil.java | 2 +-
.../src/main/java/org/apache/hudi/source/FileIndex.java | 2 +-
.../scala/org/apache/hudi/ColumnStatsIndexSupport.scala | 2 +-
.../src/main/scala/org/apache/hudi/HoodieFileIndex.scala | 2 +-
.../scala/org/apache/hudi/RecordLevelIndexSupport.scala | 2 +-
13 files changed, 18 insertions(+), 22 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index e8f327faecb..2d01f13b1db 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -2439,7 +2439,7 @@ public class HoodieWriteConfig extends HoodieConfig {
}
public boolean isRecordIndexEnabled() {
- return metadataConfig.enableRecordIndex();
+ return metadataConfig.isRecordIndexEnabled();
}
public int getRecordIndexMinFileGroupCount() {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
index dd2bda902a3..3573bf3889b 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
@@ -98,7 +98,7 @@ public class RunIndexActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I,
public RunIndexActionExecutor(HoodieEngineContext context, HoodieWriteConfig config, HoodieTable<T, I, K, O> table, String instantTime) {
super(context, config, table, instantTime);
this.txnManager = new TransactionManager(config, table.getMetaClient().getStorage());
- if (config.getMetadataConfig().enableMetrics()) {
+ if (config.getMetadataConfig().isMetricsEnabled()) {
this.metrics = Option.of(new HoodieMetadataMetrics(config.getMetricsConfig()));
} else {
this.metrics = Option.empty();
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
index 045aac6be02..a469861c8a9 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
@@ -251,7 +251,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
}
public void syncTableMetadata(HoodieWriteConfig writeConfig) {
- if (!writeConfig.getMetadataConfig().enabled()) {
+ if (!writeConfig.getMetadataConfig().isEnabled()) {
return;
}
// Open up the metadata table again, for syncing
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
index 7c6f32bc7a4..fe977aba877 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
@@ -532,7 +532,7 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
}
public void syncTableMetadata(HoodieWriteConfig writeConfig) {
- if (!writeConfig.getMetadataConfig().enabled()) {
+ if (!writeConfig.getMetadataConfig().isEnabled()) {
return;
}
// Open up the metadata table again, for syncing
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
index 5fb897c67e9..6670722bbe7 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
@@ -344,7 +344,7 @@ public final class HoodieMetadataConfig extends HoodieConfig {
return getBoolean(HoodieMetadataConfig.ASSUME_DATE_PARTITIONING);
}
- public boolean enabled() {
+ public boolean isEnabled() {
return getBoolean(ENABLE);
}
@@ -357,7 +357,7 @@ public final class HoodieMetadataConfig extends HoodieConfig {
}
public boolean isRecordIndexEnabled() {
- return getBooleanOrDefault(RECORD_INDEX_ENABLE_PROP);
+ return isEnabled() && getBooleanOrDefault(RECORD_INDEX_ENABLE_PROP);
}
public List<String> getColumnsEnabledForColumnStatsIndex() {
@@ -396,7 +396,7 @@ public final class HoodieMetadataConfig extends HoodieConfig {
return getIntOrDefault(METADATA_INDEX_CHECK_TIMEOUT_SECONDS);
}
- public boolean enableMetrics() {
+ public boolean isMetricsEnabled() {
return getBoolean(METRICS_ENABLE);
}
@@ -404,11 +404,11 @@ public final class HoodieMetadataConfig extends HoodieConfig {
return getString(DIR_FILTER_REGEX);
}
- public boolean ignoreSpuriousDeletes() {
+ public boolean shouldIgnoreSpuriousDeletes() {
return getBoolean(IGNORE_SPURIOUS_DELETES);
}
- public boolean doEnableOptimizedLogBlocksScan() {
+ public boolean isOptimizedLogBlocksScanEnabled() {
return getBoolean(ENABLE_OPTIMIZED_LOG_BLOCKS_SCAN);
}
@@ -416,10 +416,6 @@ public final class HoodieMetadataConfig extends HoodieConfig {
return getIntOrDefault(METADATA_MAX_NUM_DELTACOMMITS_WHEN_PENDING);
}
- public boolean enableRecordIndex() {
- return enabled() && getBoolean(RECORD_INDEX_ENABLE_PROP);
- }
-
public int getRecordIndexMinFileGroupCount() {
return getInt(RECORD_INDEX_MIN_FILE_GROUP_COUNT_PROP);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
index 278849600cb..513abb6364a 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
@@ -97,7 +97,7 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
this.metadataConfig = metadataConfig;
this.isMetadataTableInitialized = dataMetaClient.getTableConfig().isMetadataTableAvailable();
- if (metadataConfig.enableMetrics()) {
+ if (metadataConfig.isMetricsEnabled()) {
this.metrics = Option.of(new HoodieMetadataMetrics(HoodieMetricsConfig.newBuilder().fromProperties(metadataConfig.getProps()).build()));
} else {
this.metrics = Option.empty();
@@ -415,7 +415,7 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata {
*/
private void checkForSpuriousDeletes(HoodieMetadataPayload metadataPayload, String partitionName) {
if (!metadataPayload.getDeletions().isEmpty()) {
- if (metadataConfig.ignoreSpuriousDeletes()) {
+ if (metadataConfig.shouldIgnoreSpuriousDeletes()) {
LOG.warn("Metadata record for " + partitionName + " encountered some files to be deleted which was not added before. "
+ "Ignoring the spurious deletes as the `" + HoodieMetadataConfig.IGNORE_SPURIOUS_DELETES.key() + "` config is set to true");
} else {
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
index d2d1878afa6..7ea0bb87b73 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
@@ -495,7 +495,7 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
.withLogBlockTimestamps(validInstantTimestamps)
.enableFullScan(allowFullScan)
.withPartition(partitionName)
- .withEnableOptimizedLogBlocksScan(metadataConfig.doEnableOptimizedLogBlocksScan())
+ .withEnableOptimizedLogBlocksScan(metadataConfig.isOptimizedLogBlocksScanEnabled())
.withTableMetaClient(metadataMetaClient)
.build();
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java
index 1b3bd129432..4d39c4eef25 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java
@@ -117,7 +117,7 @@ public interface HoodieTableMetadata extends Serializable, AutoCloseable {
}
static HoodieTableMetadata create(HoodieEngineContext engineContext, HoodieMetadataConfig metadataConfig, String datasetBasePath, boolean reuse) {
- if (metadataConfig.enabled()) {
+ if (metadataConfig.isEnabled()) {
HoodieBackedTableMetadata metadata = createHoodieBackedTableMetadata(engineContext, metadataConfig, datasetBasePath, reuse);
// If the MDT is not initialized then we fallback to FSBackedTableMetadata
if (metadata.isMetadataTableInitialized()) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
index fc0720915ed..062cfedfc12 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
@@ -1700,7 +1700,7 @@ public class HoodieTableMetadataUtil {
}
// Does any enabled partition being enabled need to track the written records
- return config.enableRecordIndex();
+ return config.isRecordIndexEnabled();
}
/**
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java
index a954293e26b..a1461975811 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java
@@ -306,7 +306,7 @@ public class FileIndex {
// - Any expression not directly referencing top-level column (for ex, sub-queries, since there's
// nothing CSI in particular could be applied for)
if (dataSkippingEnabled) {
- if (metadataConfig.enabled()) {
+ if (metadataConfig.isEnabled()) {
return true;
} else {
LOG.warn("Data skipping requires Metadata Table to be enabled! Disable the data skipping");
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ColumnStatsIndexSupport.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ColumnStatsIndexSupport.scala
index f38d4318cac..7b14863ce38 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ColumnStatsIndexSupport.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ColumnStatsIndexSupport.scala
@@ -83,7 +83,7 @@ class ColumnStatsIndexSupport(spark: SparkSession,
* w/in the Metadata Table
*/
def isIndexAvailable: Boolean = {
- checkState(metadataConfig.enabled, "Metadata Table support has to be enabled")
+ checkState(metadataConfig.isEnabled, "Metadata Table support has to be enabled")
metaClient.getTableConfig.getMetadataPartitions.contains(HoodieTableMetadataUtil.PARTITION_NAME_COLUMN_STATS)
}
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala
index a15b8c7224c..243782f81f9 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala
@@ -430,7 +430,7 @@ case class HoodieFileIndex(spark: SparkSession,
private def isDataSkippingEnabled: Boolean = getConfigValue(options, spark.sessionState.conf,
DataSourceReadOptions.ENABLE_DATA_SKIPPING.key, DataSourceReadOptions.ENABLE_DATA_SKIPPING.defaultValue.toString).toBoolean
- private def isMetadataTableEnabled: Boolean = metadataConfig.enabled()
+ private def isMetadataTableEnabled: Boolean = metadataConfig.isEnabled()
private def isColumnStatsIndexEnabled: Boolean = metadataConfig.isColumnStatsIndexEnabled
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala
index 76873803955..1b72df1c97d 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala
@@ -108,7 +108,7 @@ class RecordLevelIndexSupport(spark: SparkSession,
* Return true if metadata table is enabled and record index metadata partition is available.
*/
def isIndexAvailable: Boolean = {
- metadataConfig.enabled && metaClient.getTableConfig.getMetadataPartitions.contains(HoodieTableMetadataUtil.PARTITION_NAME_RECORD_INDEX)
+ metadataConfig.isEnabled && metaClient.getTableConfig.getMetadataPartitions.contains(HoodieTableMetadataUtil.PARTITION_NAME_RECORD_INDEX)
}
}
(hudi) 15/16: [HUDI-6386] Enable testArchivalWithMultiWriters back as they are passing (#9085)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 0f6bc8cbd58a143151834f8b39c10cd4fcbf05fb
Author: Sivabalan Narayanan <n....@gmail.com>
AuthorDate: Tue Apr 23 21:19:50 2024 -0700
[HUDI-6386] Enable testArchivalWithMultiWriters back as they are passing (#9085)
Co-authored-by: Balaji Varadarajan <vb...@apache.org>
---
.../org/apache/hudi/io/TestHoodieTimelineArchiver.java | 15 +++------------
1 file changed, 3 insertions(+), 12 deletions(-)
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
index 034bcc8788a..1edef971097 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
@@ -65,7 +65,6 @@ import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
@@ -689,17 +688,9 @@ public class TestHoodieTimelineArchiver extends HoodieSparkClientTestHarness {
assertThrows(HoodieException.class, () -> metaClient.getArchivedTimeline().reload());
}
- @Disabled("HUDI-6841")
- public void testArchivalWithMultiWritersMDTDisabled() throws Exception {
- testArchivalWithMultiWriters(false);
- }
-
- @Disabled("HUDI-6386")
- public void testArchivalWithMultiWriters() throws Exception {
- testArchivalWithMultiWriters(true);
- }
-
- private void testArchivalWithMultiWriters(boolean enableMetadata) throws Exception {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testArchivalWithMultiWriters(boolean enableMetadata) throws Exception {
HoodieWriteConfig writeConfig = initTestTableAndGetWriteConfig(enableMetadata, 4, 5, 5, 2,
HoodieTableType.COPY_ON_WRITE, false, 10, 209715200,
HoodieFailedWritesCleaningPolicy.LAZY, WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL);
(hudi) 01/16: [HUDI-7640] Uses UUID as temporary file suffix for HoodieStorage.createImmutableFileInPath (#11052)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 50f367dbe1fca428b404fe04a91d7593dac0b311
Author: Danny Chan <yu...@gmail.com>
AuthorDate: Fri Apr 19 14:07:47 2024 +0800
[HUDI-7640] Uses UUID as temporary file suffix for HoodieStorage.createImmutableFileInPath (#11052)
---
.../src/main/java/org/apache/hudi/storage/HoodieStorage.java | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java b/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
index adf9371c243..be160caba3b 100644
--- a/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
+++ b/hudi-io/src/main/java/org/apache/hudi/storage/HoodieStorage.java
@@ -37,6 +37,7 @@ import java.io.OutputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
+import java.util.UUID;
/**
* Provides I/O APIs on files and directories on storage.
@@ -45,7 +46,6 @@ import java.util.List;
@PublicAPIClass(maturity = ApiMaturityLevel.EVOLVING)
public abstract class HoodieStorage implements Closeable {
public static final Logger LOG = LoggerFactory.getLogger(HoodieStorage.class);
- public static final String TMP_PATH_POSTFIX = ".tmp";
/**
* @return the scheme of the storage.
@@ -249,8 +249,11 @@ public abstract class HoodieStorage implements Closeable {
* empty, will first write the content to a temp file if {needCreateTempFile} is
* true, and then rename it back after the content is written.
*
- * @param path file path.
- * @param content content to be stored.
+ * <p>CAUTION: if this method is invoked in multi-threads for concurrent write of the same file,
+ * an existence check of the file is recommended.
+ *
+ * @param path File path.
+ * @param content Content to be stored.
*/
@PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
public final void createImmutableFileInPath(StoragePath path,
@@ -267,7 +270,7 @@ public abstract class HoodieStorage implements Closeable {
if (content.isPresent() && needTempFile) {
StoragePath parent = path.getParent();
- tmpPath = new StoragePath(parent, path.getName() + TMP_PATH_POSTFIX);
+ tmpPath = new StoragePath(parent, path.getName() + "." + UUID.randomUUID());
fsout = create(tmpPath, false);
fsout.write(content.get());
}
(hudi) 03/16: [HUDI-7643] Fix test by using the right StreamSync constructor (#11056)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 5020b2a9e18546cfe70e422f2b30b08bee9a47b5
Author: Sagar Sumit <sa...@gmail.com>
AuthorDate: Fri Apr 19 15:55:46 2024 +0530
[HUDI-7643] Fix test by using the right StreamSync constructor (#11056)
---
.../org/apache/hudi/utilities/streamer/TestStreamSyncUnitTests.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSyncUnitTests.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSyncUnitTests.java
index 8ff5b6ee933..fe775f95a36 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSyncUnitTests.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSyncUnitTests.java
@@ -141,7 +141,7 @@ public class TestStreamSyncUnitTests {
@MethodSource("getCheckpointToResumeCases")
void testGetCheckpointToResume(HoodieStreamer.Config cfg, HoodieCommitMetadata commitMetadata, Option<String> expectedResumeCheckpoint) throws IOException {
HoodieSparkEngineContext hoodieSparkEngineContext = mock(HoodieSparkEngineContext.class);
- FileSystem fs = mock(FileSystem.class);
+ HoodieStorage storage = HoodieStorageUtils.getStorage(mock(FileSystem.class));
TypedProperties props = new TypedProperties();
SparkSession sparkSession = mock(SparkSession.class);
Configuration configuration = mock(Configuration.class);
@@ -152,7 +152,7 @@ public class TestStreamSyncUnitTests {
when(commitsTimeline.lastInstant()).thenReturn(Option.of(hoodieInstant));
StreamSync streamSync = new StreamSync(cfg, sparkSession, props, hoodieSparkEngineContext,
- fs, configuration, client -> true, null,Option.empty(),null,Option.empty(),true,true);
+ storage, configuration, client -> true, null,Option.empty(),null,Option.empty(),true,true);
StreamSync spy = spy(streamSync);
doReturn(Option.of(commitMetadata)).when(spy).getLatestCommitMetadataWithValidCheckpointInfo(any());
(hudi) 16/16: [MINOR] Fix LoggerName for JDBCExecutor (#11063)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit e358279a860e77eefe52bde52af93bbbeda45f94
Author: chengabc930919 <63...@users.noreply.github.com>
AuthorDate: Wed Apr 24 13:35:26 2024 +0800
[MINOR] Fix LoggerName for JDBCExecutor (#11063)
---
.../src/main/java/org/apache/hudi/hive/ddl/JDBCExecutor.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/JDBCExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/JDBCExecutor.java
index 026bf880835..0ffcdf2a0d3 100644
--- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/JDBCExecutor.java
+++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/JDBCExecutor.java
@@ -47,7 +47,7 @@ import static org.apache.hudi.hive.util.HiveSchemaUtil.HIVE_ESCAPE_CHARACTER;
*/
public class JDBCExecutor extends QueryBasedDDLExecutor {
- private static final Logger LOG = LoggerFactory.getLogger(QueryBasedDDLExecutor.class);
+ private static final Logger LOG = LoggerFactory.getLogger(JDBCExecutor.class);
private Connection connection;
(hudi) 02/16: [HUDI-7618] Add ability to ignore checkpoints in delta streamer (#11018)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 5582cde47b7af6f18b66eff39546120d7377e3c4
Author: Sampan S Nayak <sa...@gmail.com>
AuthorDate: Fri Apr 19 11:55:43 2024 +0530
[HUDI-7618] Add ability to ignore checkpoints in delta streamer (#11018)
---
.../hudi/utilities/streamer/HoodieStreamer.java | 7 +++
.../apache/hudi/utilities/streamer/StreamSync.java | 13 ++++-
.../streamer/TestStreamSyncUnitTests.java | 61 ++++++++++++++++++++++
3 files changed, 79 insertions(+), 2 deletions(-)
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
index 643a240638c..b42b3dbeda2 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
@@ -428,6 +428,13 @@ public class HoodieStreamer implements Serializable {
@Parameter(names = {"--config-hot-update-strategy-class"}, description = "Configuration hot update in continuous mode")
public String configHotUpdateStrategyClass = "";
+ @Parameter(names = {"--ignore-checkpoint"}, description = "Set this config with a unique value, recommend using a timestamp value or UUID."
+ + " Setting this config indicates that the subsequent sync should ignore the last committed checkpoint for the source. The config value is stored"
+ + " in the commit history, so setting the config with same values would not have any affect. This config can be used in scenarios like kafka topic change,"
+ + " where we would want to start ingesting from the latest or earliest offset after switching the topic (in this case we would want to ignore the previously"
+ + " committed checkpoint, and rely on other configs to pick the starting offsets).")
+ public String ignoreCheckpoint = null;
+
public boolean isAsyncCompactionEnabled() {
return continuousMode && !forceDisableCompaction
&& HoodieTableType.MERGE_ON_READ.equals(HoodieTableType.valueOf(tableType));
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
index f1184a75abe..3c6c36d2a3e 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
@@ -164,6 +164,7 @@ public class StreamSync implements Serializable, Closeable {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(StreamSync.class);
private static final String NULL_PLACEHOLDER = "[null]";
+ public static final String CHECKPOINT_IGNORE_KEY = "deltastreamer.checkpoint.ignore_key";
/**
* Delta Sync Config.
@@ -732,7 +733,8 @@ public class StreamSync implements Serializable, Closeable {
* @return the checkpoint to resume from if applicable.
* @throws IOException
*/
- private Option<String> getCheckpointToResume(Option<HoodieTimeline> commitsTimelineOpt) throws IOException {
+ @VisibleForTesting
+ Option<String> getCheckpointToResume(Option<HoodieTimeline> commitsTimelineOpt) throws IOException {
Option<String> resumeCheckpointStr = Option.empty();
// try get checkpoint from commits(including commit and deltacommit)
// in COW migrating to MOR case, the first batch of the deltastreamer will lost the checkpoint from COW table, cause the dataloss
@@ -749,7 +751,11 @@ public class StreamSync implements Serializable, Closeable {
if (commitMetadataOption.isPresent()) {
HoodieCommitMetadata commitMetadata = commitMetadataOption.get();
LOG.debug("Checkpoint reset from metadata: " + commitMetadata.getMetadata(CHECKPOINT_RESET_KEY));
- if (cfg.checkpoint != null && (StringUtils.isNullOrEmpty(commitMetadata.getMetadata(CHECKPOINT_RESET_KEY))
+ if (cfg.ignoreCheckpoint != null && (StringUtils.isNullOrEmpty(commitMetadata.getMetadata(CHECKPOINT_IGNORE_KEY))
+ || !cfg.ignoreCheckpoint.equals(commitMetadata.getMetadata(CHECKPOINT_IGNORE_KEY)))) {
+ // we ignore any existing checkpoint and start ingesting afresh
+ resumeCheckpointStr = Option.empty();
+ } else if (cfg.checkpoint != null && (StringUtils.isNullOrEmpty(commitMetadata.getMetadata(CHECKPOINT_RESET_KEY))
|| !cfg.checkpoint.equals(commitMetadata.getMetadata(CHECKPOINT_RESET_KEY)))) {
resumeCheckpointStr = Option.of(cfg.checkpoint);
} else if (!StringUtils.isNullOrEmpty(commitMetadata.getMetadata(CHECKPOINT_KEY))) {
@@ -852,6 +858,9 @@ public class StreamSync implements Serializable, Closeable {
if (cfg.checkpoint != null) {
checkpointCommitMetadata.put(CHECKPOINT_RESET_KEY, cfg.checkpoint);
}
+ if (cfg.ignoreCheckpoint != null) {
+ checkpointCommitMetadata.put(CHECKPOINT_IGNORE_KEY, cfg.ignoreCheckpoint);
+ }
}
if (hasErrors) {
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSyncUnitTests.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSyncUnitTests.java
index c22c948e70b..8ff5b6ee933 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSyncUnitTests.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSyncUnitTests.java
@@ -22,7 +22,10 @@ package org.apache.hudi.utilities.streamer;
import org.apache.hudi.DataSourceWriteOptions;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieErrorTableConfig;
import org.apache.hudi.storage.HoodieStorage;
@@ -43,9 +46,13 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
+import java.io.IOException;
import java.util.stream.Stream;
import static org.apache.hudi.config.HoodieErrorTableConfig.ERROR_ENABLE_VALIDATE_TARGET_SCHEMA;
+import static org.apache.hudi.utilities.streamer.HoodieStreamer.CHECKPOINT_KEY;
+import static org.apache.hudi.utilities.streamer.HoodieStreamer.CHECKPOINT_RESET_KEY;
+import static org.apache.hudi.utilities.streamer.StreamSync.CHECKPOINT_IGNORE_KEY;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
@@ -130,6 +137,60 @@ public class TestStreamSyncUnitTests {
HoodieErrorTableConfig.ERROR_ENABLE_VALIDATE_TARGET_SCHEMA.defaultValue());
}
+ @ParameterizedTest
+ @MethodSource("getCheckpointToResumeCases")
+ void testGetCheckpointToResume(HoodieStreamer.Config cfg, HoodieCommitMetadata commitMetadata, Option<String> expectedResumeCheckpoint) throws IOException {
+ HoodieSparkEngineContext hoodieSparkEngineContext = mock(HoodieSparkEngineContext.class);
+ FileSystem fs = mock(FileSystem.class);
+ TypedProperties props = new TypedProperties();
+ SparkSession sparkSession = mock(SparkSession.class);
+ Configuration configuration = mock(Configuration.class);
+ HoodieTimeline commitsTimeline = mock(HoodieTimeline.class);
+ HoodieInstant hoodieInstant = mock(HoodieInstant.class);
+
+ when(commitsTimeline.filter(any())).thenReturn(commitsTimeline);
+ when(commitsTimeline.lastInstant()).thenReturn(Option.of(hoodieInstant));
+
+ StreamSync streamSync = new StreamSync(cfg, sparkSession, props, hoodieSparkEngineContext,
+ fs, configuration, client -> true, null,Option.empty(),null,Option.empty(),true,true);
+ StreamSync spy = spy(streamSync);
+ doReturn(Option.of(commitMetadata)).when(spy).getLatestCommitMetadataWithValidCheckpointInfo(any());
+
+ Option<String> resumeCheckpoint = spy.getCheckpointToResume(Option.of(commitsTimeline));
+ assertEquals(expectedResumeCheckpoint,resumeCheckpoint);
+ }
+
+ private static Stream<Arguments> getCheckpointToResumeCases() {
+ return Stream.of(
+ // Checkpoint has been manually overridden (reset-checkpoint)
+ Arguments.of(generateDeltaStreamerConfig("new-reset-checkpoint",null),generateCommitMetadata("old-reset-checkpoint",null,null),Option.of("new-reset-checkpoint")),
+ // Checkpoint not reset/ Ignored, continuing from previous run
+ Arguments.of(generateDeltaStreamerConfig("old-reset-checkpoint",null),generateCommitMetadata("old-reset-checkpoint",null,"checkpoint-prev-run"),Option.of("checkpoint-prev-run")),
+ // Checkpoint not reset/ Ignored, continuing from previous run (ignore checkpoint has not changed)
+ Arguments.of(generateDeltaStreamerConfig("old-reset-checkpoint","123445"),generateCommitMetadata("old-reset-checkpoint","123445","checkpoint-prev-run"),Option.of("checkpoint-prev-run")),
+ // Ignore checkpoint set, existing checkpoints will be ignored
+ Arguments.of(generateDeltaStreamerConfig("old-reset-checkpoint","123445"),generateCommitMetadata("old-reset-checkpoint","123422","checkpoint-prev-run"),Option.empty()),
+ // Ignore checkpoint set, existing checkpoints will be ignored (reset-checkpoint ignored)
+ Arguments.of(generateDeltaStreamerConfig("new-reset-checkpoint","123445"),generateCommitMetadata("old-reset-checkpoint","123422","checkpoint-prev-run"),Option.empty())
+ );
+ }
+
+ private static HoodieStreamer.Config generateDeltaStreamerConfig(String checkpoint, String ignoreCheckpoint) {
+ HoodieStreamer.Config cfg = new HoodieStreamer.Config();
+ cfg.checkpoint = checkpoint;
+ cfg.ignoreCheckpoint = ignoreCheckpoint;
+ cfg.tableType = "MERGE_ON_READ";
+ return cfg;
+ }
+
+ private static HoodieCommitMetadata generateCommitMetadata(String resetCheckpointValue, String ignoreCheckpointValue, String checkpointValue) {
+ HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
+ commitMetadata.addMetadata(CHECKPOINT_RESET_KEY,resetCheckpointValue);
+ commitMetadata.addMetadata(CHECKPOINT_IGNORE_KEY,ignoreCheckpointValue);
+ commitMetadata.addMetadata(CHECKPOINT_KEY,checkpointValue);
+ return commitMetadata;
+ }
+
private SchemaProvider getSchemaProvider(String name, boolean isNullTargetSchema) {
SchemaProvider schemaProvider = mock(SchemaProvider.class);
Schema sourceSchema = mock(Schema.class);
(hudi) 05/16: [MINOR] Added configurations of Hudi table, file-based SQL source, Hudi error table, and timestamp key generator to configuration listing (#11057)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 82f7485e4f9bc8fa2af53ff83229ee87b31710f0
Author: Geser Dugarov <ge...@gmail.com>
AuthorDate: Sat Apr 20 07:43:37 2024 +0700
[MINOR] Added configurations of Hudi table, file-based SQL source, Hudi error table, and timestamp key generator to configuration listing (#11057)
---
.../org/apache/hudi/config/HoodieErrorTableConfig.java | 3 ++-
.../org/apache/hudi/common/config/ConfigGroups.java | 4 ++++
.../hudi/common/config/TimestampKeyGeneratorConfig.java | 2 +-
.../org/apache/hudi/common/table/HoodieTableConfig.java | 17 ++++++++++-------
.../hudi/utilities/config/SqlFileBasedSourceConfig.java | 3 ++-
5 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieErrorTableConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieErrorTableConfig.java
index 8ba013b00ee..1db8f2c4b5f 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieErrorTableConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieErrorTableConfig.java
@@ -21,6 +21,7 @@ package org.apache.hudi.config;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
+import org.apache.hudi.common.config.HoodieConfig;
import javax.annotation.concurrent.Immutable;
@@ -30,7 +31,7 @@ import java.util.Arrays;
@ConfigClassProperty(name = "Error table Configs",
groupName = ConfigGroups.Names.WRITE_CLIENT,
description = "Configurations that are required for Error table configs")
-public class HoodieErrorTableConfig {
+public class HoodieErrorTableConfig extends HoodieConfig {
public static final ConfigProperty<Boolean> ERROR_TABLE_ENABLED = ConfigProperty
.key("hoodie.errortable.enable")
.defaultValue(false)
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java
index daba6f9203e..95a809f10ca 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/config/ConfigGroups.java
@@ -30,6 +30,7 @@ public class ConfigGroups {
* {@link ConfigGroups#getDescription}.
*/
public enum Names {
+ TABLE_CONFIG("Hudi Table Config"),
ENVIRONMENT_CONFIG("Environment Config"),
SPARK_DATASOURCE("Spark Datasource Configs"),
FLINK_SQL("Flink Sql Configs"),
@@ -94,6 +95,9 @@ public class ConfigGroups {
public static String getDescription(Names names) {
String description;
switch (names) {
+ case TABLE_CONFIG:
+ description = "Basic Hudi Table configuration parameters.";
+ break;
case ENVIRONMENT_CONFIG:
description = "Hudi supports passing configurations via a configuration file "
+ "`hudi-default.conf` in which each line consists of a key and a value "
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/TimestampKeyGeneratorConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/TimestampKeyGeneratorConfig.java
index 7098c076279..46b66371b31 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/config/TimestampKeyGeneratorConfig.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/config/TimestampKeyGeneratorConfig.java
@@ -31,7 +31,7 @@ import java.util.concurrent.TimeUnit;
+ "the partition field. The field values are interpreted as timestamps and not just "
+ "converted to string while generating partition path value for records. Record key is "
+ "same as before where it is chosen by field name.")
-public class TimestampKeyGeneratorConfig {
+public class TimestampKeyGeneratorConfig extends HoodieConfig {
private static final String TIMESTAMP_KEYGEN_CONFIG_PREFIX = "hoodie.keygen.timebased.";
@Deprecated
private static final String OLD_TIMESTAMP_KEYGEN_CONFIG_PREFIX = "hoodie.deltastreamer.keygen.timebased.";
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
index c098f483bf8..5de826992f8 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
@@ -20,6 +20,8 @@ package org.apache.hudi.common.table;
import org.apache.hudi.common.bootstrap.index.HFileBootstrapIndex;
import org.apache.hudi.common.bootstrap.index.NoOpBootstrapIndex;
+import org.apache.hudi.common.config.ConfigClassProperty;
+import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.config.OrderedProperties;
@@ -48,6 +50,8 @@ import org.apache.avro.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.concurrent.Immutable;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -71,13 +75,12 @@ import static org.apache.hudi.common.config.TimestampKeyGeneratorConfig.TIMESTAM
import static org.apache.hudi.common.config.TimestampKeyGeneratorConfig.TIMESTAMP_TIMEZONE_FORMAT;
import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
-/**
- * Configurations on the Hoodie Table like type of ingestion, storage formats, hive table name etc Configurations are loaded from hoodie.properties, these properties are usually set during
- * initializing a path as hoodie base path and never changes during the lifetime of a hoodie table.
- *
- * @see HoodieTableMetaClient
- * @since 0.3.0
- */
+@Immutable
+@ConfigClassProperty(name = "Hudi Table Basic Configs",
+ groupName = ConfigGroups.Names.TABLE_CONFIG,
+ description = "Configurations of the Hudi Table like type of ingestion, storage formats, hive table name etc."
+ + " Configurations are loaded from hoodie.properties, these properties are usually set during"
+ + " initializing a path as hoodie base path and never changes during the lifetime of a hoodie table.")
public class HoodieTableConfig extends HoodieConfig {
private static final Logger LOG = LoggerFactory.getLogger(HoodieTableConfig.class);
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/config/SqlFileBasedSourceConfig.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/config/SqlFileBasedSourceConfig.java
index 2eaf64a4a4f..413ba1a3643 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/config/SqlFileBasedSourceConfig.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/config/SqlFileBasedSourceConfig.java
@@ -22,6 +22,7 @@ package org.apache.hudi.utilities.config;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.ConfigProperty;
+import org.apache.hudi.common.config.HoodieConfig;
import javax.annotation.concurrent.Immutable;
@@ -33,7 +34,7 @@ import static org.apache.hudi.common.util.ConfigUtils.STREAMER_CONFIG_PREFIX;
groupName = ConfigGroups.Names.HUDI_STREAMER,
subGroupName = ConfigGroups.SubGroupNames.DELTA_STREAMER_SOURCE,
description = "Configurations controlling the behavior of File-based SQL Source in Hudi Streamer.")
-public class SqlFileBasedSourceConfig {
+public class SqlFileBasedSourceConfig extends HoodieConfig {
public static final ConfigProperty<String> SOURCE_SQL_FILE = ConfigProperty
.key(STREAMER_CONFIG_PREFIX + "source.sql.file")
(hudi) 06/16: [HUDI-7628] Rename FSUtils.getPartitionPath to constructAbsolutePath (#11054)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 541b4dcf2441d8bc9233d415b5de9622faa01e80
Author: Vova Kolmakov <wo...@gmail.com>
AuthorDate: Wed May 15 00:48:27 2024 -0700
[HUDI-7628] Rename FSUtils.getPartitionPath to constructAbsolutePath (#11054)
Co-authored-by: Vova Kolmakov <ko...@huawei-partners.com>
---
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 4 +--
.../apache/hudi/cli/commands/RepairsCommand.java | 4 +--
.../apache/hudi/client/CompactionAdminClient.java | 4 +--
.../index/bucket/ConsistentBucketIndexUtils.java | 8 +++---
.../org/apache/hudi/io/HoodieAppendHandle.java | 2 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 2 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 2 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 4 +--
.../metadata/HoodieBackedTableMetadataWriter.java | 2 +-
.../hudi/table/action/compact/HoodieCompactor.java | 2 +-
.../table/action/rollback/BaseRollbackHelper.java | 4 +--
.../rollback/ListingBasedRollbackStrategy.java | 6 ++--
.../rollback/MarkerBasedRollbackStrategy.java | 2 +-
.../marker/TimelineServerBasedWriteMarkers.java | 4 +--
.../org/apache/hudi/table/marker/WriteMarkers.java | 2 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 4 +--
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 4 +--
.../TestSavepointRestoreMergeOnRead.java | 8 +++---
.../java/org/apache/hudi/table/TestCleaner.java | 4 +--
.../hudi/table/marker/TestWriteMarkersBase.java | 2 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 32 +++++++++++-----------
.../hudi/common/model/CompactionOperation.java | 2 +-
.../hudi/common/model/HoodieCommitMetadata.java | 8 +++---
.../hudi/common/table/cdc/HoodieCDCExtractor.java | 4 +--
.../clean/CleanMetadataV1MigrationHandler.java | 2 +-
.../clean/CleanPlanV2MigrationHandler.java | 2 +-
.../compaction/CompactionV1MigrationHandler.java | 2 +-
.../table/view/AbstractTableFileSystemView.java | 4 +--
.../IncrementalTimelineSyncFileSystemView.java | 2 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 2 +-
.../org/apache/hudi/IncrementalRelation.scala | 2 +-
.../RepairAddpartitionmetaProcedure.scala | 2 +-
.../RepairMigratePartitionMetaProcedure.scala | 2 +-
.../procedures/ShowInvalidParquetProcedure.scala | 2 +-
.../TestSparkConsistentBucketClustering.java | 2 +-
.../apache/hudi/sync/adb/HoodieAdbJdbcClient.java | 10 +++----
.../org/apache/hudi/hive/ddl/HMSDDLExecutor.java | 4 +--
.../hudi/hive/ddl/QueryBasedDDLExecutor.java | 4 +--
.../org/apache/hudi/hive/TestHiveSyncTool.java | 2 +-
.../apache/hudi/sync/common/HoodieSyncClient.java | 4 +--
.../hudi/utilities/HoodieDataTableUtils.java | 2 +-
.../utilities/HoodieMetadataTableValidator.java | 8 +++---
.../hudi/utilities/HoodieSnapshotCopier.java | 4 +--
.../hudi/utilities/HoodieSnapshotExporter.java | 4 +--
44 files changed, 93 insertions(+), 93 deletions(-)
diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
index 9e3c088f8b0..11e3eaea1c0 100644
--- a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
+++ b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
@@ -301,7 +301,7 @@ public class AWSGlueCatalogSyncClient extends HoodieSyncClient {
try {
StorageDescriptor sd = table.storageDescriptor();
List<PartitionInput> partitionInputList = partitionsToAdd.stream().map(partition -> {
- String fullPartitionPath = FSUtils.getPartitionPathInHadoopPath(s3aToS3(getBasePath()), partition).toString();
+ String fullPartitionPath = FSUtils.constructAbsolutePathInHadoopPath(s3aToS3(getBasePath()), partition).toString();
List<String> partitionValues = partitionValueExtractor.extractPartitionValuesInPath(partition);
StorageDescriptor partitionSD = sd.copy(copySd -> copySd.location(fullPartitionPath));
return PartitionInput.builder().values(partitionValues).storageDescriptor(partitionSD).build();
@@ -345,7 +345,7 @@ public class AWSGlueCatalogSyncClient extends HoodieSyncClient {
try {
StorageDescriptor sd = table.storageDescriptor();
List<BatchUpdatePartitionRequestEntry> updatePartitionEntries = changedPartitions.stream().map(partition -> {
- String fullPartitionPath = FSUtils.getPartitionPathInHadoopPath(s3aToS3(getBasePath()), partition).toString();
+ String fullPartitionPath = FSUtils.constructAbsolutePathInHadoopPath(s3aToS3(getBasePath()), partition).toString();
List<String> partitionValues = partitionValueExtractor.extractPartitionValuesInPath(partition);
StorageDescriptor partitionSD = sd.copy(copySd -> copySd.location(fullPartitionPath));
PartitionInput partitionInput = PartitionInput.builder().values(partitionValues).storageDescriptor(partitionSD).build();
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
index 28e1a0d39ba..0eedbf964fe 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
@@ -128,7 +128,7 @@ public class RepairsCommand {
int ind = 0;
for (String partition : partitionPaths) {
- StoragePath partitionPath = FSUtils.getPartitionPath(basePath, partition);
+ StoragePath partitionPath = FSUtils.constructAbsolutePath(basePath, partition);
String[] row = new String[3];
row[0] = partition;
row[1] = "Yes";
@@ -236,7 +236,7 @@ public class RepairsCommand {
int ind = 0;
for (String partitionPath : partitionPaths) {
StoragePath partition =
- FSUtils.getPartitionPath(client.getBasePath(), partitionPath);
+ FSUtils.constructAbsolutePath(client.getBasePath(), partitionPath);
Option<StoragePath> textFormatFile =
HoodiePartitionMetadata.textFormatMetaPathIfExists(HoodieCLI.storage, partition);
Option<StoragePath> baseFormatFile =
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
index a63524dfbb5..dbe07b7d0f3 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
@@ -296,7 +296,7 @@ public class CompactionAdminClient extends BaseHoodieClient {
if (operation.getDataFileName().isPresent()) {
String expPath = metaClient.getStorage()
.getPathInfo(new StoragePath(
- FSUtils.getPartitionPath(metaClient.getBasePath(), operation.getPartitionPath()),
+ FSUtils.constructAbsolutePath(metaClient.getBasePath(), operation.getPartitionPath()),
operation.getDataFileName().get()))
.getPath().toString();
ValidationUtils.checkArgument(df.isPresent(),
@@ -309,7 +309,7 @@ public class CompactionAdminClient extends BaseHoodieClient {
try {
List<StoragePathInfo> pathInfoList = metaClient.getStorage()
.listDirectEntries(new StoragePath(
- FSUtils.getPartitionPath(metaClient.getBasePath(), operation.getPartitionPath()), dp));
+ FSUtils.constructAbsolutePath(metaClient.getBasePath(), operation.getPartitionPath()), dp));
ValidationUtils.checkArgument(pathInfoList.size() == 1, "Expect only 1 file-status");
return new HoodieLogFile(pathInfoList.get(0));
} catch (FileNotFoundException fe) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
index 7a124d25ee9..a90e0db6a06 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIndexUtils.java
@@ -108,8 +108,8 @@ public class ConsistentBucketIndexUtils {
*/
public static Option<HoodieConsistentHashingMetadata> loadMetadata(HoodieTable table, String partition) {
HoodieTableMetaClient metaClient = table.getMetaClient();
- Path metadataPath = FSUtils.getPartitionPathInHadoopPath(metaClient.getHashingMetadataPath(), partition);
- Path partitionPath = FSUtils.getPartitionPathInHadoopPath(metaClient.getBasePathV2().toString(), partition);
+ Path metadataPath = FSUtils.constructAbsolutePathInHadoopPath(metaClient.getHashingMetadataPath(), partition);
+ Path partitionPath = FSUtils.constructAbsolutePathInHadoopPath(metaClient.getBasePathV2().toString(), partition);
try {
Predicate<FileStatus> hashingMetaCommitFilePredicate = fileStatus -> {
String filename = fileStatus.getPath().getName();
@@ -186,7 +186,7 @@ public class ConsistentBucketIndexUtils {
*/
public static boolean saveMetadata(HoodieTable table, HoodieConsistentHashingMetadata metadata, boolean overwrite) {
HoodieStorage storage = table.getMetaClient().getStorage();
- StoragePath dir = FSUtils.getPartitionPath(
+ StoragePath dir = FSUtils.constructAbsolutePath(
table.getMetaClient().getHashingMetadataPath(), metadata.getPartitionPath());
StoragePath fullPath = new StoragePath(dir, metadata.getFilename());
try (OutputStream out = storage.create(fullPath, overwrite)) {
@@ -267,7 +267,7 @@ public class ConsistentBucketIndexUtils {
* @return true if hashing metadata file is latest else false
*/
private static boolean recommitMetadataFile(HoodieTable table, FileStatus metaFile, String partition) {
- Path partitionPath = new Path(FSUtils.getPartitionPath(table.getMetaClient().getBasePathV2(), partition).toUri());
+ Path partitionPath = new Path(FSUtils.constructAbsolutePath(table.getMetaClient().getBasePathV2(), partition).toUri());
String timestamp = getTimestampFromFile(metaFile.getPath().getName());
if (table.getPendingCommitTimeline().containsInstant(timestamp)) {
return false;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
index 2bac318fc81..5b414c79b53 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
@@ -206,7 +206,7 @@ public class HoodieAppendHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O
try {
// Save hoodie partition meta in the partition path
HoodiePartitionMetadata partitionMetadata = new HoodiePartitionMetadata(storage, baseInstantTime,
- new StoragePath(config.getBasePath()), FSUtils.getPartitionPath(config.getBasePath(), partitionPath),
+ new StoragePath(config.getBasePath()), FSUtils.constructAbsolutePath(config.getBasePath(), partitionPath),
hoodieTable.getPartitionMetafileFormat());
partitionMetadata.trySave();
this.writer = createLogWriter(fileSlice, baseInstantTime);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
index 0ad4e212a1a..ce908f89bb6 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
@@ -98,7 +98,7 @@ public class HoodieCreateHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O
try {
HoodiePartitionMetadata partitionMetadata = new HoodiePartitionMetadata(storage, instantTime,
new StoragePath(config.getBasePath()),
- FSUtils.getPartitionPath(config.getBasePath(), partitionPath),
+ FSUtils.constructAbsolutePath(config.getBasePath(), partitionPath),
hoodieTable.getPartitionMetafileFormat());
partitionMetadata.trySave();
createMarkerFile(partitionPath,
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
index afae82fd13f..797684b71af 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
@@ -174,7 +174,7 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
HoodiePartitionMetadata partitionMetadata = new HoodiePartitionMetadata(storage, instantTime,
new StoragePath(config.getBasePath()),
- FSUtils.getPartitionPath(config.getBasePath(), partitionPath),
+ FSUtils.constructAbsolutePath(config.getBasePath(), partitionPath),
hoodieTable.getPartitionMetafileFormat());
partitionMetadata.trySave();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
index de45c51ecf1..486102b5222 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
@@ -114,7 +114,7 @@ public abstract class HoodieWriteHandle<T, I, K, O> extends HoodieIOHandle<T, I,
}
public StoragePath makeNewPath(String partitionPath) {
- StoragePath path = FSUtils.getPartitionPath(config.getBasePath(), partitionPath);
+ StoragePath path = FSUtils.constructAbsolutePath(config.getBasePath(), partitionPath);
try {
if (!storage.exists(path)) {
storage.createDirectory(path); // create a new partition as needed.
@@ -247,7 +247,7 @@ public abstract class HoodieWriteHandle<T, I, K, O> extends HoodieIOHandle<T, I,
: Option.empty();
return HoodieLogFormat.newWriterBuilder()
- .onParentPath(FSUtils.getPartitionPath(hoodieTable.getMetaClient().getBasePath(), partitionPath))
+ .onParentPath(FSUtils.constructAbsolutePath(hoodieTable.getMetaClient().getBasePath(), partitionPath))
.withFileId(fileId)
.overBaseCommit(baseCommitTime)
.withLogVersion(latestLogFile.map(HoodieLogFile::getLogVersion).orElse(HoodieLogFile.LOGFILE_BASE_VERSION))
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 7a084aba52c..0714f27d0e8 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -713,7 +713,7 @@ public abstract class HoodieBackedTableMetadataWriter<I> implements HoodieTableM
final HoodieDeleteBlock block = new HoodieDeleteBlock(new DeleteRecord[0], blockHeader);
try (HoodieLogFormat.Writer writer = HoodieLogFormat.newWriterBuilder()
- .onParentPath(FSUtils.getPartitionPath(metadataWriteConfig.getBasePath(), metadataPartition.getPartitionPath()))
+ .onParentPath(FSUtils.constructAbsolutePath(metadataWriteConfig.getBasePath(), metadataPartition.getPartitionPath()))
.withFileId(fileGroupFileId)
.overBaseCommit(instantTime)
.withLogVersion(HoodieLogFile.LOGFILE_BASE_VERSION)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java
index 9ede03b12cd..9e38410fed9 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java
@@ -186,7 +186,7 @@ public abstract class HoodieCompactor<T, I, K, O> implements Serializable {
LOG.info("MaxMemoryPerCompaction => " + maxMemoryPerCompaction);
List<String> logFiles = operation.getDeltaFileNames().stream().map(p ->
- new StoragePath(FSUtils.getPartitionPath(
+ new StoragePath(FSUtils.constructAbsolutePath(
metaClient.getBasePath(), operation.getPartitionPath()), p).toString())
.collect(toList());
HoodieMergedLogRecordScanner scanner = HoodieMergedLogRecordScanner.newBuilder()
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java
index 7d16726c20d..f9cff041e9a 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java
@@ -165,7 +165,7 @@ public class BaseRollbackHelper implements Serializable {
WriteMarkers writeMarkers = WriteMarkersFactory.get(config.getMarkersType(), table, instantTime);
writer = HoodieLogFormat.newWriterBuilder()
- .onParentPath(FSUtils.getPartitionPath(metaClient.getBasePathV2().toString(), partitionPath))
+ .onParentPath(FSUtils.constructAbsolutePath(metaClient.getBasePathV2().toString(), partitionPath))
.withFileId(fileId)
.overBaseCommit(latestBaseInstant)
.withStorage(metaClient.getStorage())
@@ -203,7 +203,7 @@ public class BaseRollbackHelper implements Serializable {
// With listing based rollback, sometimes we only get the fileID of interest(so that we can add rollback command block) w/o the actual file name.
// So, we want to ignore such invalid files from this list before we add it to the rollback stats.
- String partitionFullPath = FSUtils.getPartitionPath(metaClient.getBasePathV2().toString(), rollbackRequest.getPartitionPath()).toString();
+ String partitionFullPath = FSUtils.constructAbsolutePath(metaClient.getBasePathV2().toString(), rollbackRequest.getPartitionPath()).toString();
Map<String, Long> validLogBlocksToDelete = new HashMap<>();
rollbackRequest.getLogBlocksToBeDeleted().entrySet().stream().forEach((kv) -> {
String logFileFullPath = kv.getKey();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
index 83d5d88c28f..1fd054b9407 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
@@ -225,7 +225,7 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
}
return false;
};
- return fs.listStatus(FSUtils.getPartitionPathInHadoopPath(config.getBasePath(), partitionPath), filter);
+ return fs.listStatus(FSUtils.constructAbsolutePathInHadoopPath(config.getBasePath(), partitionPath), filter);
}
private FileStatus[] fetchFilesFromInstant(HoodieInstant instantToRollback, String partitionPath, String basePath,
@@ -286,7 +286,7 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
}
private static Path[] listFilesToBeDeleted(String basePath, String partitionPath) {
- return new Path[] {FSUtils.getPartitionPathInHadoopPath(basePath, partitionPath)};
+ return new Path[] {FSUtils.constructAbsolutePathInHadoopPath(basePath, partitionPath)};
}
private static Path[] getFilesFromCommitMetadata(String basePath, HoodieCommitMetadata commitMetadata, String partitionPath) {
@@ -356,7 +356,7 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
FileSlice latestFileSlice = latestFileSlices.get(writeStat.getFileId());
String fileId = writeStat.getFileId();
String latestBaseInstant = latestFileSlice.getBaseInstantTime();
- Path fullLogFilePath = FSUtils.getPartitionPathInHadoopPath(table.getConfig().getBasePath(), writeStat.getPath());
+ Path fullLogFilePath = FSUtils.constructAbsolutePathInHadoopPath(table.getConfig().getBasePath(), writeStat.getPath());
Map<String, Long> logFilesWithBlocksToRollback = Collections.singletonMap(
fullLogFilePath.toString(), writeStat.getTotalWriteBytes() > 0 ? writeStat.getTotalWriteBytes() : 1L);
hoodieRollbackRequests.add(new HoodieRollbackRequest(partitionPath, fileId, latestBaseInstant,
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/MarkerBasedRollbackStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/MarkerBasedRollbackStrategy.java
index 648d05da61f..5ba61b38803 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/MarkerBasedRollbackStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/MarkerBasedRollbackStrategy.java
@@ -121,7 +121,7 @@ public class MarkerBasedRollbackStrategy<T, I, K, O> implements BaseRollbackPlan
LOG.warn("Find old marker type for log file: " + fileNameWithPartitionToRollback);
fileId = FSUtils.getFileIdFromFilePath(fullLogFilePath);
baseCommitTime = FSUtils.getCommitTime(fullLogFilePath.getName());
- StoragePath partitionPath = FSUtils.getPartitionPath(config.getBasePath(), relativePartitionPath);
+ StoragePath partitionPath = FSUtils.constructAbsolutePath(config.getBasePath(), relativePartitionPath);
// NOTE: Since we're rolling back incomplete Delta Commit, it only could have appended its
// block to the latest log-file
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java
index 7b0fda4ea47..f738449d7dc 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java
@@ -158,7 +158,7 @@ public class TimelineServerBasedWriteMarkers extends WriteMarkers {
LOG.info("[timeline-server-based] Created marker file " + partitionPath + "/" + markerFileName
+ " in " + timer.endTimer() + " ms");
if (success) {
- return Option.of(new StoragePath(FSUtils.getPartitionPath(markerDirPath, partitionPath), markerFileName));
+ return Option.of(new StoragePath(FSUtils.constructAbsolutePath(markerDirPath, partitionPath), markerFileName));
} else {
return Option.empty();
}
@@ -177,7 +177,7 @@ public class TimelineServerBasedWriteMarkers extends WriteMarkers {
+ " in " + timer.endTimer() + " ms");
if (success) {
- return Option.of(new StoragePath(FSUtils.getPartitionPath(markerDirPath, partitionPath), markerFileName));
+ return Option.of(new StoragePath(FSUtils.constructAbsolutePath(markerDirPath, partitionPath), markerFileName));
} else {
// this failed may due to early conflict detection, so we need to throw out.
throw new HoodieEarlyConflictDetectionException(new ConcurrentModificationException("Early conflict detected but cannot resolve conflicts for overlapping writes"));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java
index e481d0b9e4b..cd9f67b5b20 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java
@@ -182,7 +182,7 @@ public abstract class WriteMarkers implements Serializable {
* @return path of the marker file
*/
protected StoragePath getMarkerPath(String partitionPath, String fileName, IOType type) {
- StoragePath path = FSUtils.getPartitionPath(markerDirPath, partitionPath);
+ StoragePath path = FSUtils.constructAbsolutePath(markerDirPath, partitionPath);
String markerFileName = getMarkerFileName(fileName, type);
return new StoragePath(path, markerFileName);
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
index 19455773153..4227e14165f 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
@@ -104,7 +104,7 @@ public class HoodieRowDataCreateHandle implements Serializable {
storage,
instantTime,
new StoragePath(writeConfig.getBasePath()),
- FSUtils.getPartitionPath(writeConfig.getBasePath(), partitionPath),
+ FSUtils.constructAbsolutePath(writeConfig.getBasePath(), partitionPath),
table.getPartitionMetafileFormat());
partitionMetadata.trySave();
createMarkerFile(partitionPath, FSUtils.makeBaseFileName(this.instantTime, getWriteToken(), this.fileId, table.getBaseFileExtension()));
@@ -190,7 +190,7 @@ public class HoodieRowDataCreateHandle implements Serializable {
private Path makeNewPath(String partitionPath) {
StoragePath path =
- FSUtils.getPartitionPath(writeConfig.getBasePath(), partitionPath);
+ FSUtils.constructAbsolutePath(writeConfig.getBasePath(), partitionPath);
try {
if (!storage.exists(path)) {
storage.createDirectory(path); // create a new partition as needed.
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
index 890b12899f1..0d164f379fe 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
@@ -141,7 +141,7 @@ public class HoodieRowCreateHandle implements Serializable {
storage,
instantTime,
new StoragePath(writeConfig.getBasePath()),
- FSUtils.getPartitionPath(writeConfig.getBasePath(), partitionPath),
+ FSUtils.constructAbsolutePath(writeConfig.getBasePath(), partitionPath),
table.getPartitionMetafileFormat());
partitionMetadata.trySave();
@@ -262,7 +262,7 @@ public class HoodieRowCreateHandle implements Serializable {
}
private static Path makeNewPath(FileSystem fs, String partitionPath, String fileName, HoodieWriteConfig writeConfig) {
- Path path = FSUtils.getPartitionPathInHadoopPath(writeConfig.getBasePath(), partitionPath);
+ Path path = FSUtils.constructAbsolutePathInHadoopPath(writeConfig.getBasePath(), partitionPath);
try {
if (!fs.exists(path)) {
fs.mkdirs(path); // create a new partition as needed.
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestSavepointRestoreMergeOnRead.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestSavepointRestoreMergeOnRead.java
index 04f931904bd..5027170cca7 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestSavepointRestoreMergeOnRead.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestSavepointRestoreMergeOnRead.java
@@ -119,7 +119,7 @@ public class TestSavepointRestoreMergeOnRead extends HoodieClientTestBase {
StoragePathFilter filter = (path) -> path.toString().contains(finalCompactionCommit);
for (String pPath : dataGen.getPartitionPaths()) {
assertEquals(0, storage.listDirectEntries(
- FSUtils.getPartitionPath(hoodieWriteConfig.getBasePath(), pPath),
+ FSUtils.constructAbsolutePath(hoodieWriteConfig.getBasePath(), pPath),
filter).size());
}
}
@@ -164,7 +164,7 @@ public class TestSavepointRestoreMergeOnRead extends HoodieClientTestBase {
StoragePathFilter filter = (path) -> path.toString().contains(secondCommit);
for (String pPath : dataGen.getPartitionPaths()) {
assertEquals(1, storage.listDirectEntries(
- FSUtils.getPartitionPath(hoodieWriteConfig.getBasePath(), pPath), filter)
+ FSUtils.constructAbsolutePath(hoodieWriteConfig.getBasePath(), pPath), filter)
.size());
}
@@ -203,7 +203,7 @@ public class TestSavepointRestoreMergeOnRead extends HoodieClientTestBase {
filter = (path) -> path.toString().contains(secondCommit);
for (String pPath : dataGen.getPartitionPaths()) {
assertEquals(0, storage.listDirectEntries(
- FSUtils.getPartitionPath(hoodieWriteConfig.getBasePath(), pPath), filter)
+ FSUtils.constructAbsolutePath(hoodieWriteConfig.getBasePath(), pPath), filter)
.size());
}
// ensure files matching 1st commit is intact
@@ -211,7 +211,7 @@ public class TestSavepointRestoreMergeOnRead extends HoodieClientTestBase {
for (String pPath : dataGen.getPartitionPaths()) {
assertEquals(1,
storage.listDirectEntries(
- FSUtils.getPartitionPath(hoodieWriteConfig.getBasePath(), pPath),
+ FSUtils.constructAbsolutePath(hoodieWriteConfig.getBasePath(), pPath),
filter).size());
}
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
index b9a289ec5e4..a41b76387a6 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
@@ -861,9 +861,9 @@ public class TestCleaner extends HoodieCleanerTestBase {
version2Plan.getFilePathsToBeDeletedPerPartition().get(partition1).size());
assertEquals(version1Plan.getFilesToBeDeletedPerPartition().get(partition2).size(),
version2Plan.getFilePathsToBeDeletedPerPartition().get(partition2).size());
- assertEquals(new Path(FSUtils.getPartitionPathInHadoopPath(metaClient.getBasePath(), partition1), fileName1).toString(),
+ assertEquals(new Path(FSUtils.constructAbsolutePathInHadoopPath(metaClient.getBasePath(), partition1), fileName1).toString(),
version2Plan.getFilePathsToBeDeletedPerPartition().get(partition1).get(0).getFilePath());
- assertEquals(new Path(FSUtils.getPartitionPathInHadoopPath(metaClient.getBasePath(), partition2), fileName2).toString(),
+ assertEquals(new Path(FSUtils.constructAbsolutePathInHadoopPath(metaClient.getBasePath(), partition2), fileName2).toString(),
version2Plan.getFilePathsToBeDeletedPerPartition().get(partition2).get(0).getFilePath());
// Downgrade and verify version 1 plan
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestWriteMarkersBase.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestWriteMarkersBase.java
index 037613eaa5a..7eba0f31ca8 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestWriteMarkersBase.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestWriteMarkersBase.java
@@ -58,7 +58,7 @@ public abstract class TestWriteMarkersBase extends HoodieCommonTestHarness {
}
private void createInvalidFile(String partitionPath, String invalidFileName) {
- StoragePath path = FSUtils.getPartitionPath(markerFolderPath, partitionPath);
+ StoragePath path = FSUtils.constructAbsolutePath(markerFolderPath, partitionPath);
StoragePath invalidFilePath = new StoragePath(path, invalidFileName);
try {
storage.create(invalidFilePath, false).close();
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
index ebc71aa2ac0..0685d8d4a88 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
@@ -704,40 +704,40 @@ public class FSUtils {
return sizeInBytes / (1024 * 1024);
}
- public static Path getPartitionPathInHadoopPath(String basePath, String partitionPath) {
- if (StringUtils.isNullOrEmpty(partitionPath)) {
+ public static Path constructAbsolutePathInHadoopPath(String basePath, String relativePartitionPath) {
+ if (StringUtils.isNullOrEmpty(relativePartitionPath)) {
return new Path(basePath);
}
// NOTE: We have to chop leading "/" to make sure Hadoop does not treat it like
// absolute path
- String properPartitionPath = partitionPath.startsWith("/")
- ? partitionPath.substring(1)
- : partitionPath;
- return getPartitionPath(new CachingPath(basePath), properPartitionPath);
+ String properPartitionPath = relativePartitionPath.startsWith(PATH_SEPARATOR)
+ ? relativePartitionPath.substring(1)
+ : relativePartitionPath;
+ return constructAbsolutePath(new CachingPath(basePath), properPartitionPath);
}
- public static StoragePath getPartitionPath(String basePath, String partitionPath) {
- if (StringUtils.isNullOrEmpty(partitionPath)) {
+ public static StoragePath constructAbsolutePath(String basePath, String relativePartitionPath) {
+ if (StringUtils.isNullOrEmpty(relativePartitionPath)) {
return new StoragePath(basePath);
}
// NOTE: We have to chop leading "/" to make sure Hadoop does not treat it like
// absolute path
- String properPartitionPath = partitionPath.startsWith("/")
- ? partitionPath.substring(1)
- : partitionPath;
- return getPartitionPath(new StoragePath(basePath), properPartitionPath);
+ String properPartitionPath = relativePartitionPath.startsWith(PATH_SEPARATOR)
+ ? relativePartitionPath.substring(1)
+ : relativePartitionPath;
+ return constructAbsolutePath(new StoragePath(basePath), properPartitionPath);
}
- public static Path getPartitionPath(Path basePath, String partitionPath) {
+ public static Path constructAbsolutePath(Path basePath, String relativePartitionPath) {
// For non-partitioned table, return only base-path
- return StringUtils.isNullOrEmpty(partitionPath) ? basePath : new CachingPath(basePath, partitionPath);
+ return StringUtils.isNullOrEmpty(relativePartitionPath) ? basePath : new CachingPath(basePath, relativePartitionPath);
}
- public static StoragePath getPartitionPath(StoragePath basePath, String partitionPath) {
+ public static StoragePath constructAbsolutePath(StoragePath basePath, String relativePartitionPath) {
// For non-partitioned table, return only base-path
- return StringUtils.isNullOrEmpty(partitionPath) ? basePath : new StoragePath(basePath, partitionPath);
+ return StringUtils.isNullOrEmpty(relativePartitionPath) ? basePath : new StoragePath(basePath, relativePartitionPath);
}
/**
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/CompactionOperation.java b/hudi-common/src/main/java/org/apache/hudi/common/model/CompactionOperation.java
index 04aceb336f9..15accbd49c2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/CompactionOperation.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/CompactionOperation.java
@@ -119,7 +119,7 @@ public class CompactionOperation implements Serializable {
public Option<HoodieBaseFile> getBaseFile(String basePath, String partitionPath) {
Option<BaseFile> externalBaseFile = bootstrapFilePath.map(BaseFile::new);
- StoragePath dirPath = FSUtils.getPartitionPath(basePath, partitionPath);
+ StoragePath dirPath = FSUtils.constructAbsolutePath(basePath, partitionPath);
return dataFileName.map(df -> {
return externalBaseFile.map(ext -> new HoodieBaseFile(new StoragePath(dirPath, df).toString(), ext))
.orElseGet(() -> new HoodieBaseFile(new StoragePath(dirPath, df).toString()));
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java
index eeb16cf12af..b371c6acad1 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java
@@ -135,7 +135,7 @@ public class HoodieCommitMetadata implements Serializable {
HashMap<String, String> fullPaths = new HashMap<>();
for (Map.Entry<String, String> entry : getFileIdAndRelativePaths().entrySet()) {
String fullPath = entry.getValue() != null
- ? FSUtils.getPartitionPath(basePath, entry.getValue()).toString()
+ ? FSUtils.constructAbsolutePath(basePath, entry.getValue()).toString()
: null;
fullPaths.put(entry.getKey(), fullPath);
}
@@ -147,7 +147,7 @@ public class HoodieCommitMetadata implements Serializable {
if (getPartitionToWriteStats().get(partitionPath) != null) {
for (HoodieWriteStat stat : getPartitionToWriteStats().get(partitionPath)) {
if ((stat.getFileId() != null)) {
- String fullPath = FSUtils.getPartitionPathInHadoopPath(basePath, stat.getPath()).toString();
+ String fullPath = FSUtils.constructAbsolutePathInHadoopPath(basePath, stat.getPath()).toString();
fullPaths.add(fullPath);
}
}
@@ -184,7 +184,7 @@ public class HoodieCommitMetadata implements Serializable {
for (HoodieWriteStat stat : stats) {
String relativeFilePath = stat.getPath();
StoragePath fullPath = relativeFilePath != null
- ? FSUtils.getPartitionPath(basePath, relativeFilePath) : null;
+ ? FSUtils.constructAbsolutePath(basePath, relativeFilePath) : null;
if (fullPath != null) {
long blockSize =
HoodieStorageUtils.getStorage(fullPath.toString(), hadoopConf).getDefaultBlockSize(fullPath);
@@ -218,7 +218,7 @@ public class HoodieCommitMetadata implements Serializable {
for (HoodieWriteStat stat : stats) {
String relativeFilePath = stat.getPath();
StoragePath fullPath =
- relativeFilePath != null ? FSUtils.getPartitionPath(basePath,
+ relativeFilePath != null ? FSUtils.constructAbsolutePath(basePath,
relativeFilePath) : null;
if (fullPath != null) {
StoragePathInfo pathInfo =
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCExtractor.java b/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCExtractor.java
index eea2ebbbc81..fc838bcc1e5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCExtractor.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCExtractor.java
@@ -184,7 +184,7 @@ public class HoodieCDCExtractor {
try {
List<StoragePathInfo> touchedFiles = new ArrayList<>();
for (String touchedPartition : touchedPartitions) {
- StoragePath partitionPath = FSUtils.getPartitionPath(basePath, touchedPartition);
+ StoragePath partitionPath = FSUtils.constructAbsolutePath(basePath, touchedPartition);
touchedFiles.addAll(storage.listDirectEntries(partitionPath));
}
return new HoodieTableFileSystemView(
@@ -313,7 +313,7 @@ public class HoodieCDCExtractor {
HoodieFileGroupId fgId,
HoodieInstant instant,
String currentLogFile) {
- StoragePath partitionPath = FSUtils.getPartitionPath(basePath, fgId.getPartitionPath());
+ StoragePath partitionPath = FSUtils.constructAbsolutePath(basePath, fgId.getPartitionPath());
if (instant.getAction().equals(DELTA_COMMIT_ACTION)) {
String currentLogFileName = new StoragePath(currentLogFile).getName();
Option<Pair<String, List<String>>> fileSliceOpt =
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataV1MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataV1MigrationHandler.java
index 1f7b5792eb0..41e3dc79399 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataV1MigrationHandler.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanMetadataV1MigrationHandler.java
@@ -99,6 +99,6 @@ public class CleanMetadataV1MigrationHandler extends AbstractMigratorBase<Hoodie
return fileName;
}
- return new Path(FSUtils.getPartitionPath(basePath, partitionPath), fileName).toString();
+ return new Path(FSUtils.constructAbsolutePath(basePath, partitionPath), fileName).toString();
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanPlanV2MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanPlanV2MigrationHandler.java
index 7317991af37..99b5185ba73 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanPlanV2MigrationHandler.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/clean/CleanPlanV2MigrationHandler.java
@@ -55,7 +55,7 @@ public class CleanPlanV2MigrationHandler extends AbstractMigratorBase<HoodieClea
Map<String, List<HoodieCleanFileInfo>> filePathsPerPartition =
plan.getFilesToBeDeletedPerPartition().entrySet().stream().map(e -> Pair.of(e.getKey(), e.getValue().stream()
.map(v -> new HoodieCleanFileInfo(
- new Path(FSUtils.getPartitionPathInHadoopPath(metaClient.getBasePath(), e.getKey()), v).toString(), false))
+ new Path(FSUtils.constructAbsolutePathInHadoopPath(metaClient.getBasePath(), e.getKey()), v).toString(), false))
.collect(Collectors.toList()))).collect(Collectors.toMap(Pair::getKey, Pair::getValue));
return new HoodieCleanerPlan(plan.getEarliestInstantToRetain(), plan.getLastCompletedCommitTimestamp(),
plan.getPolicy(), new HashMap<>(), VERSION, filePathsPerPartition, new ArrayList<>(), Collections.emptyMap());
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/compaction/CompactionV1MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/compaction/CompactionV1MigrationHandler.java
index 17488a637ce..31905b1ad4b 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/compaction/CompactionV1MigrationHandler.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/compaction/CompactionV1MigrationHandler.java
@@ -78,6 +78,6 @@ public class CompactionV1MigrationHandler extends AbstractMigratorBase<HoodieCom
return fileName;
}
- return new Path(FSUtils.getPartitionPath(basePath, partitionPath), fileName).toString();
+ return new Path(FSUtils.constructAbsolutePath(basePath, partitionPath), fileName).toString();
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
index d7097aed170..049af4f420c 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
@@ -348,7 +348,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
// Pairs of relative partition path and absolute partition path
List<Pair<String, StoragePath>> absolutePartitionPathList = partitionSet.stream()
.map(partition -> Pair.of(
- partition, FSUtils.getPartitionPath(metaClient.getBasePathV2(), partition)))
+ partition, FSUtils.constructAbsolutePath(metaClient.getBasePathV2(), partition)))
.collect(Collectors.toList());
long beginLsTs = System.currentTimeMillis();
Map<Pair<String, StoragePath>, List<StoragePathInfo>> pathInfoMap =
@@ -420,7 +420,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
*/
private List<StoragePathInfo> getAllFilesInPartition(String relativePartitionPath)
throws IOException {
- StoragePath partitionPath = FSUtils.getPartitionPath(metaClient.getBasePathV2(),
+ StoragePath partitionPath = FSUtils.constructAbsolutePath(metaClient.getBasePathV2(),
relativePartitionPath);
long beginLsTs = System.currentTimeMillis();
List<StoragePathInfo> pathInfoList = listPartition(partitionPath);
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java
index 410f13b2b29..42888e2ad8a 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java
@@ -364,7 +364,7 @@ public abstract class IncrementalTimelineSyncFileSystemView extends AbstractTabl
final String partitionPath = entry.getValue().getPartitionPath();
List<String> fullPathList = entry.getValue().getSuccessDeleteFiles()
.stream().map(fileName -> new StoragePath(FSUtils
- .getPartitionPathInHadoopPath(basePath, partitionPath).toString(), fileName).toString())
+ .constructAbsolutePathInHadoopPath(basePath, partitionPath).toString(), fileName).toString())
.collect(Collectors.toList());
removeFileSlicesForPartition(timeline, instant, entry.getKey(), fullPathList);
});
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java
index f8091d8dc36..ac4d2ea7783 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/ITTestHoodieFlinkCompactor.java
@@ -427,7 +427,7 @@ public class ITTestHoodieFlinkCompactor {
FSUtils.getAllPartitionPaths(HoodieFlinkEngineContext.DEFAULT, metaClient.getBasePath(), false, false).forEach(
partition -> {
try {
- storage.listDirectEntries(FSUtils.getPartitionPath(metaClient.getBasePathV2(), partition))
+ storage.listDirectEntries(FSUtils.constructAbsolutePath(metaClient.getBasePathV2(), partition))
.stream()
.filter(f -> FSUtils.isBaseFile(new Path(f.getPath().toUri())))
.forEach(f -> {
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelation.scala
index d83e4172556..cb5803dfe5e 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelation.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelation.scala
@@ -161,7 +161,7 @@ class IncrementalRelation(val sqlContext: SQLContext,
fromBytes(metaClient.getActiveTimeline.getInstantDetails(instant).get, classOf[HoodieReplaceCommitMetadata])
replaceMetadata.getPartitionToReplaceFileIds.entrySet().flatMap { entry =>
entry.getValue.map { e =>
- val fullPath = FSUtils.getPartitionPath(basePath, entry.getKey).toString
+ val fullPath = FSUtils.constructAbsolutePath(basePath, entry.getKey).toString
(e, fullPath)
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala
index 03ef6cc3f54..3ae183101e8 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala
@@ -62,7 +62,7 @@ class RepairAddpartitionmetaProcedure extends BaseProcedure with ProcedureBuilde
val rows = new util.ArrayList[Row](partitionPaths.size)
for (partition <- partitionPaths) {
- val partitionPath: StoragePath = FSUtils.getPartitionPath(basePath, partition)
+ val partitionPath: StoragePath = FSUtils.constructAbsolutePath(basePath, partition)
var isPresent = "Yes"
var action = "None"
if (!HoodiePartitionMetadata.hasPartitionMetadata(metaClient.getStorage, partitionPath)) {
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
index 07fd7c92a68..4edb95c0cfc 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
@@ -67,7 +67,7 @@ class RepairMigratePartitionMetaProcedure extends BaseProcedure with ProcedureBu
val rows = new util.ArrayList[Row](partitionPaths.size)
for (partitionPath <- partitionPaths) {
- val partition: StoragePath = FSUtils.getPartitionPath(tablePath, partitionPath)
+ val partition: StoragePath = FSUtils.constructAbsolutePath(tablePath, partitionPath)
val textFormatFile: Option[StoragePath] = HoodiePartitionMetadata.textFormatMetaPathIfExists(
metaClient.getStorage, partition)
val baseFormatFile: Option[StoragePath] = HoodiePartitionMetadata.baseFormatMetaPathIfExists(
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInvalidParquetProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInvalidParquetProcedure.scala
index 0abb050ca2b..8758537a800 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInvalidParquetProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInvalidParquetProcedure.scala
@@ -53,7 +53,7 @@ class ShowInvalidParquetProcedure extends BaseProcedure with ProcedureBuilder {
val serHadoopConf = new SerializableConfiguration(jsc.hadoopConfiguration())
javaRdd.rdd.map(part => {
val fs = HadoopFSUtils.getFs(new Path(srcPath), serHadoopConf.get())
- FSUtils.getAllDataFilesInPartition(fs, FSUtils.getPartitionPathInHadoopPath(srcPath, part))
+ FSUtils.getAllDataFilesInPartition(fs, FSUtils.constructAbsolutePathInHadoopPath(srcPath, part))
}).flatMap(_.toList)
.filter(status => {
val filePath = status.getPath
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java
index 5910bcb0899..96e4a8f0ce4 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkConsistentBucketClustering.java
@@ -189,7 +189,7 @@ public class TestSparkConsistentBucketClustering extends HoodieSparkClientTestHa
Arrays.stream(dataGen.getPartitionPaths()).forEach(p -> {
if (!isCommitFilePresent) {
StoragePath metadataPath =
- FSUtils.getPartitionPath(table.getMetaClient().getHashingMetadataPath(), p);
+ FSUtils.constructAbsolutePath(table.getMetaClient().getHashingMetadataPath(), p);
try {
table.getMetaClient().getStorage().listDirectEntries(metadataPath).forEach(fl -> {
if (fl.getPath().getName()
diff --git a/hudi-sync/hudi-adb-sync/src/main/java/org/apache/hudi/sync/adb/HoodieAdbJdbcClient.java b/hudi-sync/hudi-adb-sync/src/main/java/org/apache/hudi/sync/adb/HoodieAdbJdbcClient.java
index 2c557c35f76..0c4305017f1 100644
--- a/hudi-sync/hudi-adb-sync/src/main/java/org/apache/hudi/sync/adb/HoodieAdbJdbcClient.java
+++ b/hudi-sync/hudi-adb-sync/src/main/java/org/apache/hudi/sync/adb/HoodieAdbJdbcClient.java
@@ -323,7 +323,7 @@ public class HoodieAdbJdbcClient extends HoodieSyncClient {
if (!StringUtils.isNullOrEmpty(str)) {
List<String> values = partitionValueExtractor.extractPartitionValuesInPath(str);
Path storagePartitionPath =
- FSUtils.getPartitionPathInHadoopPath(config.getString(META_SYNC_BASE_PATH), String.join("/", values));
+ FSUtils.constructAbsolutePathInHadoopPath(config.getString(META_SYNC_BASE_PATH), String.join("/", values));
String fullStoragePartitionPath =
Path.getPathWithoutSchemeAndAuthority(storagePartitionPath).toUri().getPath();
partitions.put(values, fullStoragePartitionPath);
@@ -359,7 +359,7 @@ public class HoodieAdbJdbcClient extends HoodieSyncClient {
.append(tableName).append("`").append(" add if not exists ");
for (String partition : partitions) {
String partitionClause = getPartitionClause(partition);
- Path partitionPath = FSUtils.getPartitionPathInHadoopPath(config.getString(META_SYNC_BASE_PATH), partition);
+ Path partitionPath = FSUtils.constructAbsolutePathInHadoopPath(config.getString(META_SYNC_BASE_PATH), partition);
String fullPartitionPathStr = config.generateAbsolutePathStr(partitionPath);
sqlBuilder.append(" partition (").append(partitionClause).append(") location '")
.append(fullPartitionPathStr).append("' ");
@@ -376,7 +376,7 @@ public class HoodieAdbJdbcClient extends HoodieSyncClient {
String alterTable = "alter table `" + tableName + "`";
for (String partition : partitions) {
String partitionClause = getPartitionClause(partition);
- Path partitionPath = FSUtils.getPartitionPathInHadoopPath(config.getString(META_SYNC_BASE_PATH), partition);
+ Path partitionPath = FSUtils.constructAbsolutePathInHadoopPath(config.getString(META_SYNC_BASE_PATH), partition);
String fullPartitionPathStr = config.generateAbsolutePathStr(partitionPath);
String changePartition = alterTable + " add if not exists partition (" + partitionClause
+ ") location '" + fullPartitionPathStr + "'";
@@ -455,13 +455,13 @@ public class HoodieAdbJdbcClient extends HoodieSyncClient {
List<PartitionEvent> events = new ArrayList<>();
for (String storagePartition : partitionStoragePartitions) {
Path storagePartitionPath =
- FSUtils.getPartitionPathInHadoopPath(config.getString(META_SYNC_BASE_PATH), storagePartition);
+ FSUtils.constructAbsolutePathInHadoopPath(config.getString(META_SYNC_BASE_PATH), storagePartition);
String fullStoragePartitionPath = Path.getPathWithoutSchemeAndAuthority(storagePartitionPath).toUri().getPath();
// Check if the partition values or if hdfs path is the same
List<String> storagePartitionValues = partitionValueExtractor.extractPartitionValuesInPath(storagePartition);
if (config.getBoolean(ADB_SYNC_USE_HIVE_STYLE_PARTITIONING)) {
String partition = String.join("/", storagePartitionValues);
- storagePartitionPath = FSUtils.getPartitionPathInHadoopPath(config.getString(META_SYNC_BASE_PATH), partition);
+ storagePartitionPath = FSUtils.constructAbsolutePathInHadoopPath(config.getString(META_SYNC_BASE_PATH), partition);
fullStoragePartitionPath = Path.getPathWithoutSchemeAndAuthority(storagePartitionPath).toUri().getPath();
}
if (!storagePartitionValues.isEmpty()) {
diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java
index 2f82aa2c006..b5471079524 100644
--- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java
+++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java
@@ -205,7 +205,7 @@ public class HMSDDLExecutor implements DDLExecutor {
partitionSd.setOutputFormat(sd.getOutputFormat());
partitionSd.setSerdeInfo(sd.getSerdeInfo());
String fullPartitionPath =
- FSUtils.getPartitionPathInHadoopPath(syncConfig.getString(META_SYNC_BASE_PATH), x).toString();
+ FSUtils.constructAbsolutePathInHadoopPath(syncConfig.getString(META_SYNC_BASE_PATH), x).toString();
List<String> partitionValues = partitionValueExtractor.extractPartitionValuesInPath(x);
partitionSd.setLocation(fullPartitionPath);
partitionList.add(new Partition(partitionValues, databaseName, tableName, 0, 0, partitionSd, null));
@@ -229,7 +229,7 @@ public class HMSDDLExecutor implements DDLExecutor {
try {
StorageDescriptor sd = client.getTable(databaseName, tableName).getSd();
List<Partition> partitionList = changedPartitions.stream().map(partition -> {
- Path partitionPath = FSUtils.getPartitionPathInHadoopPath(syncConfig.getString(META_SYNC_BASE_PATH), partition);
+ Path partitionPath = FSUtils.constructAbsolutePathInHadoopPath(syncConfig.getString(META_SYNC_BASE_PATH), partition);
String partitionScheme = partitionPath.toUri().getScheme();
String fullPartitionPath = StorageSchemes.HDFS.getScheme().equals(partitionScheme)
? FSUtils.getDFSFullPartitionPath(syncConfig.getHadoopFileSystem(), partitionPath) : partitionPath.toString();
diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java
index e3b2b913944..194f99705bf 100644
--- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java
+++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/QueryBasedDDLExecutor.java
@@ -162,7 +162,7 @@ public abstract class QueryBasedDDLExecutor implements DDLExecutor {
for (int i = 0; i < partitions.size(); i++) {
String partitionClause = getPartitionClause(partitions.get(i));
String fullPartitionPath =
- FSUtils.getPartitionPathInHadoopPath(config.getString(META_SYNC_BASE_PATH), partitions.get(i)).toString();
+ FSUtils.constructAbsolutePathInHadoopPath(config.getString(META_SYNC_BASE_PATH), partitions.get(i)).toString();
alterSQL.append(" PARTITION (").append(partitionClause).append(") LOCATION '").append(fullPartitionPath)
.append("' ");
if ((i + 1) % batchSyncPartitionNum == 0) {
@@ -211,7 +211,7 @@ public abstract class QueryBasedDDLExecutor implements DDLExecutor {
String alterTable = "ALTER TABLE " + HIVE_ESCAPE_CHARACTER + tableName + HIVE_ESCAPE_CHARACTER;
for (String partition : partitions) {
String partitionClause = getPartitionClause(partition);
- Path partitionPath = FSUtils.getPartitionPathInHadoopPath(config.getString(META_SYNC_BASE_PATH), partition);
+ Path partitionPath = FSUtils.constructAbsolutePathInHadoopPath(config.getString(META_SYNC_BASE_PATH), partition);
String partitionScheme = partitionPath.toUri().getScheme();
String fullPartitionPath = StorageSchemes.HDFS.getScheme().equals(partitionScheme)
? FSUtils.getDFSFullPartitionPath(config.getHadoopFileSystem(), partitionPath) : partitionPath.toString();
diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
index ef9d43794d6..a755c5ba4f2 100644
--- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
+++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
@@ -357,7 +357,7 @@ public class TestHiveSyncTool {
// it and generate a partition update event for it.
ddlExecutor.runSQL("ALTER TABLE `" + HiveTestUtil.TABLE_NAME
+ "` PARTITION (`datestr`='2050-01-01') SET LOCATION '"
- + FSUtils.getPartitionPathInHadoopPath(basePath, "2050/1/1").toString() + "'");
+ + FSUtils.constructAbsolutePathInHadoopPath(basePath, "2050/1/1").toString() + "'");
hivePartitions = hiveClient.getAllPartitions(HiveTestUtil.TABLE_NAME);
List<String> writtenPartitionsSince = hiveClient.getWrittenPartitionsSince(Option.empty(), Option.empty());
diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java
index 582f8ec2999..b2c26781d21 100644
--- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java
+++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java
@@ -161,7 +161,7 @@ public abstract class HoodieSyncClient implements HoodieMetaSyncOperations, Auto
List<PartitionEvent> events = new ArrayList<>();
for (String storagePartition : allPartitionsOnStorage) {
Path storagePartitionPath =
- FSUtils.getPartitionPathInHadoopPath(config.getString(META_SYNC_BASE_PATH), storagePartition);
+ FSUtils.constructAbsolutePathInHadoopPath(config.getString(META_SYNC_BASE_PATH), storagePartition);
String fullStoragePartitionPath = Path.getPathWithoutSchemeAndAuthority(storagePartitionPath).toUri().getPath();
// Check if the partition values or if hdfs path is the same
List<String> storagePartitionValues = partitionValueExtractor.extractPartitionValuesInPath(storagePartition);
@@ -205,7 +205,7 @@ public abstract class HoodieSyncClient implements HoodieMetaSyncOperations, Auto
List<PartitionEvent> events = new ArrayList<>();
for (String storagePartition : writtenPartitionsOnStorage) {
Path storagePartitionPath =
- FSUtils.getPartitionPathInHadoopPath(config.getString(META_SYNC_BASE_PATH), storagePartition);
+ FSUtils.constructAbsolutePathInHadoopPath(config.getString(META_SYNC_BASE_PATH), storagePartition);
String fullStoragePartitionPath = Path.getPathWithoutSchemeAndAuthority(storagePartitionPath).toUri().getPath();
// Check if the partition values or if hdfs path is the same
List<String> storagePartitionValues = partitionValueExtractor.extractPartitionValuesInPath(storagePartition);
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableUtils.java
index 64079f18380..7647f93c899 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableUtils.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableUtils.java
@@ -38,7 +38,7 @@ public class HoodieDataTableUtils {
String basePath) throws IOException {
List<String> allPartitionPaths = tableMetadata.getAllPartitionPaths()
.stream().map(partitionPath ->
- FSUtils.getPartitionPathInHadoopPath(basePath, partitionPath).toString())
+ FSUtils.constructAbsolutePathInHadoopPath(basePath, partitionPath).toString())
.collect(Collectors.toList());
return tableMetadata.getAllFilesInPartitions(allPartitionPaths).values().stream()
.map(fileStatuses ->
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
index 992d3e0fd16..8a2ded37fd5 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
@@ -664,7 +664,7 @@ public class HoodieMetadataTableValidator implements Serializable {
@VisibleForTesting
Option<String> getPartitionCreationInstant(HoodieStorage storage, String basePath, String partition) {
HoodiePartitionMetadata hoodiePartitionMetadata =
- new HoodiePartitionMetadata(storage, FSUtils.getPartitionPath(basePath, partition));
+ new HoodiePartitionMetadata(storage, FSUtils.constructAbsolutePath(basePath, partition));
return hoodiePartitionMetadata.readPartitionCreatedCommitTime();
}
@@ -681,7 +681,7 @@ public class HoodieMetadataTableValidator implements Serializable {
// ignore partitions created by uncommitted ingestion.
return allPartitionPathsFromFS.stream().parallel().filter(part -> {
HoodiePartitionMetadata hoodiePartitionMetadata =
- new HoodiePartitionMetadata(storage, FSUtils.getPartitionPath(basePath, part));
+ new HoodiePartitionMetadata(storage, FSUtils.constructAbsolutePath(basePath, part));
Option<String> instantOption = hoodiePartitionMetadata.readPartitionCreatedCommitTime();
if (instantOption.isPresent()) {
String instantTime = instantOption.get();
@@ -1403,7 +1403,7 @@ public class HoodieMetadataTableValidator implements Serializable {
return baseFileNameList.stream().flatMap(filename ->
new ParquetUtils().readRangeFromParquetMetadata(
metaClient.getHadoopConf(),
- new StoragePath(FSUtils.getPartitionPath(metaClient.getBasePathV2(), partitionPath), filename),
+ new StoragePath(FSUtils.constructAbsolutePath(metaClient.getBasePathV2(), partitionPath), filename),
allColumnNameList).stream())
.sorted(new HoodieColumnRangeMetadataComparator())
.collect(Collectors.toList());
@@ -1445,7 +1445,7 @@ public class HoodieMetadataTableValidator implements Serializable {
private Option<BloomFilterData> readBloomFilterFromFile(String partitionPath, String filename) {
StoragePath path = new StoragePath(
- FSUtils.getPartitionPath(metaClient.getBasePathV2(), partitionPath).toString(), filename);
+ FSUtils.constructAbsolutePath(metaClient.getBasePathV2(), partitionPath).toString(), filename);
BloomFilter bloomFilter;
HoodieConfig hoodieConfig = new HoodieConfig();
hoodieConfig.setValue(HoodieReaderConfig.USE_NATIVE_HFILE_READER,
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java
index 68567b290fd..b7dcacb97e3 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java
@@ -125,7 +125,7 @@ public class HoodieSnapshotCopier implements Serializable {
// also need to copy over partition metadata
StoragePath partitionMetaFile = HoodiePartitionMetadata.getPartitionMetafilePath(storage1,
- FSUtils.getPartitionPath(baseDir, partition)).get();
+ FSUtils.constructAbsolutePath(baseDir, partition)).get();
if (storage1.exists(partitionMetaFile)) {
filePaths.add(new Tuple2<>(partition, partitionMetaFile.toString()));
}
@@ -136,7 +136,7 @@ public class HoodieSnapshotCopier implements Serializable {
context.foreach(filesToCopy, tuple -> {
String partition = tuple._1();
Path sourceFilePath = new Path(tuple._2());
- Path toPartitionPath = FSUtils.getPartitionPathInHadoopPath(outputDir, partition);
+ Path toPartitionPath = FSUtils.constructAbsolutePathInHadoopPath(outputDir, partition);
FileSystem ifs = HadoopFSUtils.getFs(baseDir, serConf.newCopy());
if (!ifs.exists(toPartitionPath)) {
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
index c3bedcfc46a..ca94de1ff44 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
@@ -216,7 +216,7 @@ public class HoodieSnapshotExporter {
// also need to copy over partition metadata
HoodieStorage storage = HoodieStorageUtils.getStorage(cfg.sourceBasePath, serConf.newCopy());
StoragePath partitionMetaFile = HoodiePartitionMetadata.getPartitionMetafilePath(storage,
- FSUtils.getPartitionPath(cfg.sourceBasePath, partition)).get();
+ FSUtils.constructAbsolutePath(cfg.sourceBasePath, partition)).get();
if (storage.exists(partitionMetaFile)) {
filePaths.add(Pair.of(partition, partitionMetaFile.toString()));
}
@@ -226,7 +226,7 @@ public class HoodieSnapshotExporter {
context.foreach(partitionAndFileList, partitionAndFile -> {
String partition = partitionAndFile.getLeft();
Path sourceFilePath = new Path(partitionAndFile.getRight());
- Path toPartitionPath = FSUtils.getPartitionPathInHadoopPath(cfg.targetOutputPath, partition);
+ Path toPartitionPath = FSUtils.constructAbsolutePathInHadoopPath(cfg.targetOutputPath, partition);
FileSystem executorSourceFs = HadoopFSUtils.getFs(cfg.sourceBasePath, serConf.newCopy());
FileSystem executorOutputFs = HadoopFSUtils.getFs(cfg.targetOutputPath, serConf.newCopy());
(hudi) 14/16: [HUDI-7647] READ_UTC_TIMEZONE doesn't affect log files for MOR tables (#11066)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 92425585e596b38972dbad0fa74e5457e7dc9a9d
Author: Марк Бухнер <66...@users.noreply.github.com>
AuthorDate: Wed Apr 24 08:06:25 2024 +0700
[HUDI-7647] READ_UTC_TIMEZONE doesn't affect log files for MOR tables (#11066)
---
.../hudi/source/stats/ColumnStatsIndices.java | 2 +-
.../table/format/mor/MergeOnReadInputFormat.java | 8 ++---
.../apache/hudi/util/AvroToRowDataConverters.java | 42 +++++++++++++---------
.../apache/hudi/table/ITTestHoodieDataSource.java | 31 ++++++++--------
4 files changed, 46 insertions(+), 37 deletions(-)
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStatsIndices.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStatsIndices.java
index 05931876603..7032f299368 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStatsIndices.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStatsIndices.java
@@ -272,7 +272,7 @@ public class ColumnStatsIndices {
LogicalType logicalType,
Map<LogicalType, AvroToRowDataConverters.AvroToRowDataConverter> converters) {
AvroToRowDataConverters.AvroToRowDataConverter converter =
- converters.computeIfAbsent(logicalType, k -> AvroToRowDataConverters.createConverter(logicalType));
+ converters.computeIfAbsent(logicalType, k -> AvroToRowDataConverters.createConverter(logicalType, true));
return converter.convert(rawVal);
}
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java
index 29bb0a06d8c..3690fc911d8 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java
@@ -351,7 +351,7 @@ public class MergeOnReadInputFormat
final Schema requiredSchema = new Schema.Parser().parse(tableState.getRequiredAvroSchema());
final GenericRecordBuilder recordBuilder = new GenericRecordBuilder(requiredSchema);
final AvroToRowDataConverters.AvroToRowDataConverter avroToRowDataConverter =
- AvroToRowDataConverters.createRowConverter(tableState.getRequiredRowType());
+ AvroToRowDataConverters.createRowConverter(tableState.getRequiredRowType(), conf.getBoolean(FlinkOptions.READ_UTC_TIMEZONE));
final HoodieMergedLogRecordScanner scanner = FormatUtils.logScanner(split, tableSchema, internalSchemaManager.getQuerySchema(), conf, hadoopConf);
final Iterator<String> logRecordsKeyIterator = scanner.getRecords().keySet().iterator();
final int[] pkOffset = tableState.getPkOffsetsInRequired();
@@ -431,7 +431,7 @@ public class MergeOnReadInputFormat
final Schema requiredSchema = new Schema.Parser().parse(tableState.getRequiredAvroSchema());
final GenericRecordBuilder recordBuilder = new GenericRecordBuilder(requiredSchema);
final AvroToRowDataConverters.AvroToRowDataConverter avroToRowDataConverter =
- AvroToRowDataConverters.createRowConverter(tableState.getRequiredRowType());
+ AvroToRowDataConverters.createRowConverter(tableState.getRequiredRowType(), conf.getBoolean(FlinkOptions.READ_UTC_TIMEZONE));
final FormatUtils.BoundedMemoryRecords records = new FormatUtils.BoundedMemoryRecords(split, tableSchema, internalSchemaManager.getQuerySchema(), hadoopConf, conf);
final Iterator<HoodieRecord<?>> recordsIterator = records.getRecordsIterator();
@@ -478,7 +478,7 @@ public class MergeOnReadInputFormat
protected ClosableIterator<RowData> getFullLogFileIterator(MergeOnReadInputSplit split) {
final Schema tableSchema = new Schema.Parser().parse(tableState.getAvroSchema());
final AvroToRowDataConverters.AvroToRowDataConverter avroToRowDataConverter =
- AvroToRowDataConverters.createRowConverter(tableState.getRowType());
+ AvroToRowDataConverters.createRowConverter(tableState.getRowType(), conf.getBoolean(FlinkOptions.READ_UTC_TIMEZONE));
final HoodieMergedLogRecordScanner scanner = FormatUtils.logScanner(split, tableSchema, InternalSchema.getEmptyInternalSchema(), conf, hadoopConf);
final Iterator<String> logRecordsKeyIterator = scanner.getRecords().keySet().iterator();
@@ -736,7 +736,7 @@ public class MergeOnReadInputFormat
this.operationPos = operationPos;
this.avroProjection = avroProjection;
this.rowDataToAvroConverter = RowDataToAvroConverters.createConverter(tableRowType, flinkConf.getBoolean(FlinkOptions.WRITE_UTC_TIMEZONE));
- this.avroToRowDataConverter = AvroToRowDataConverters.createRowConverter(requiredRowType);
+ this.avroToRowDataConverter = AvroToRowDataConverters.createRowConverter(requiredRowType, flinkConf.getBoolean(FlinkOptions.READ_UTC_TIMEZONE));
this.projection = projection;
this.instantRange = split.getInstantRange().orElse(null);
List<String> mergers = Arrays.stream(flinkConf.getString(FlinkOptions.RECORD_MERGER_IMPLS).split(","))
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/AvroToRowDataConverters.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/AvroToRowDataConverters.java
index 38633b8ad9e..0caafca8259 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/AvroToRowDataConverters.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/AvroToRowDataConverters.java
@@ -43,6 +43,7 @@ import org.joda.time.DateTimeFieldType;
import java.io.Serializable;
import java.lang.reflect.Array;
import java.nio.ByteBuffer;
+import java.sql.Timestamp;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalTime;
@@ -72,12 +73,15 @@ public class AvroToRowDataConverters {
// -------------------------------------------------------------------------------------
// Runtime Converters
// -------------------------------------------------------------------------------------
-
public static AvroToRowDataConverter createRowConverter(RowType rowType) {
+ return createRowConverter(rowType, true);
+ }
+
+ public static AvroToRowDataConverter createRowConverter(RowType rowType, boolean utcTimezone) {
final AvroToRowDataConverter[] fieldConverters =
rowType.getFields().stream()
.map(RowType.RowField::getType)
- .map(AvroToRowDataConverters::createNullableConverter)
+ .map(type -> AvroToRowDataConverters.createNullableConverter(type, utcTimezone))
.toArray(AvroToRowDataConverter[]::new);
final int arity = rowType.getFieldCount();
@@ -94,8 +98,8 @@ public class AvroToRowDataConverters {
/**
* Creates a runtime converter which is null safe.
*/
- private static AvroToRowDataConverter createNullableConverter(LogicalType type) {
- final AvroToRowDataConverter converter = createConverter(type);
+ private static AvroToRowDataConverter createNullableConverter(LogicalType type, boolean utcTimezone) {
+ final AvroToRowDataConverter converter = createConverter(type, utcTimezone);
return avroObject -> {
if (avroObject == null) {
return null;
@@ -107,7 +111,7 @@ public class AvroToRowDataConverters {
/**
* Creates a runtime converter which assuming input object is not null.
*/
- public static AvroToRowDataConverter createConverter(LogicalType type) {
+ public static AvroToRowDataConverter createConverter(LogicalType type, boolean utcTimezone) {
switch (type.getTypeRoot()) {
case NULL:
return avroObject -> null;
@@ -129,9 +133,9 @@ public class AvroToRowDataConverters {
case TIME_WITHOUT_TIME_ZONE:
return AvroToRowDataConverters::convertToTime;
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
- return createTimestampConverter(((LocalZonedTimestampType) type).getPrecision());
+ return createTimestampConverter(((LocalZonedTimestampType) type).getPrecision(), true);
case TIMESTAMP_WITHOUT_TIME_ZONE:
- return createTimestampConverter(((TimestampType) type).getPrecision());
+ return createTimestampConverter(((TimestampType) type).getPrecision(), utcTimezone);
case CHAR:
case VARCHAR:
return avroObject -> StringData.fromString(avroObject.toString());
@@ -141,12 +145,12 @@ public class AvroToRowDataConverters {
case DECIMAL:
return createDecimalConverter((DecimalType) type);
case ARRAY:
- return createArrayConverter((ArrayType) type);
+ return createArrayConverter((ArrayType) type, utcTimezone);
case ROW:
- return createRowConverter((RowType) type);
+ return createRowConverter((RowType) type, utcTimezone);
case MAP:
case MULTISET:
- return createMapConverter(type);
+ return createMapConverter(type, utcTimezone);
default:
throw new UnsupportedOperationException("Unsupported type: " + type);
}
@@ -170,9 +174,9 @@ public class AvroToRowDataConverters {
};
}
- private static AvroToRowDataConverter createArrayConverter(ArrayType arrayType) {
+ private static AvroToRowDataConverter createArrayConverter(ArrayType arrayType, boolean utcTimezone) {
final AvroToRowDataConverter elementConverter =
- createNullableConverter(arrayType.getElementType());
+ createNullableConverter(arrayType.getElementType(), utcTimezone);
final Class<?> elementClass =
LogicalTypeUtils.toInternalConversionClass(arrayType.getElementType());
@@ -187,11 +191,11 @@ public class AvroToRowDataConverters {
};
}
- private static AvroToRowDataConverter createMapConverter(LogicalType type) {
+ private static AvroToRowDataConverter createMapConverter(LogicalType type, boolean utcTimezone) {
final AvroToRowDataConverter keyConverter =
- createConverter(DataTypes.STRING().getLogicalType());
+ createConverter(DataTypes.STRING().getLogicalType(), utcTimezone);
final AvroToRowDataConverter valueConverter =
- createNullableConverter(AvroSchemaConverter.extractValueTypeToAvroMap(type));
+ createNullableConverter(AvroSchemaConverter.extractValueTypeToAvroMap(type), utcTimezone);
return avroObject -> {
final Map<?, ?> map = (Map<?, ?>) avroObject;
@@ -205,7 +209,7 @@ public class AvroToRowDataConverters {
};
}
- private static AvroToRowDataConverter createTimestampConverter(int precision) {
+ private static AvroToRowDataConverter createTimestampConverter(int precision, boolean utcTimezone) {
final ChronoUnit chronoUnit;
if (precision <= 3) {
chronoUnit = ChronoUnit.MILLIS;
@@ -233,7 +237,11 @@ public class AvroToRowDataConverters {
"Unexpected object type for TIMESTAMP logical type. Received: " + avroObject);
}
}
- return TimestampData.fromInstant(instant);
+ if (utcTimezone) {
+ return TimestampData.fromInstant(instant);
+ } else {
+ return TimestampData.fromTimestamp(Timestamp.from(instant)); // this applies the local timezone
+ }
};
}
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
index 9be2090f5bc..bc6a250eb8c 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
@@ -479,7 +479,7 @@ public class ITTestHoodieDataSource {
}
@ParameterizedTest
- @MethodSource("tableTypeAndPartitioningParams")
+ @MethodSource("tableTypeAndBooleanTrueFalseParams")
void testStreamReadFilterByPartition(HoodieTableType tableType, boolean hiveStylePartitioning) throws Exception {
Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath());
conf.setString(FlinkOptions.TABLE_NAME, "t1");
@@ -567,7 +567,7 @@ public class ITTestHoodieDataSource {
}
@ParameterizedTest
- @MethodSource("tableTypeAndPartitioningParams")
+ @MethodSource("tableTypeAndBooleanTrueFalseParams")
void testWriteAndReadWithProctimeSequence(HoodieTableType tableType, boolean hiveStylePartitioning) {
TableEnvironment tableEnv = batchTableEnv;
String hoodieTableDDL = sql("t1")
@@ -590,7 +590,7 @@ public class ITTestHoodieDataSource {
}
@ParameterizedTest
- @MethodSource("tableTypeAndPartitioningParams")
+ @MethodSource("tableTypeAndBooleanTrueFalseParams")
void testWriteAndReadWithProctimeSequenceWithTsColumnExisting(HoodieTableType tableType, boolean hiveStylePartitioning) {
TableEnvironment tableEnv = batchTableEnv;
String hoodieTableDDL = sql("t1")
@@ -640,7 +640,7 @@ public class ITTestHoodieDataSource {
}
@ParameterizedTest
- @MethodSource("tableTypeAndPartitioningParams")
+ @MethodSource("tableTypeAndBooleanTrueFalseParams")
void testBatchModeUpsert(HoodieTableType tableType, boolean hiveStylePartitioning) {
TableEnvironment tableEnv = batchTableEnv;
String hoodieTableDDL = sql("t1")
@@ -1834,8 +1834,8 @@ public class ITTestHoodieDataSource {
}
@ParameterizedTest
- @EnumSource(value = HoodieTableType.class)
- void testWriteReadWithTimestampWithoutTZ(HoodieTableType tableType) {
+ @MethodSource("tableTypeAndBooleanTrueFalseParams")
+ void testWriteReadWithTimestampWithoutTZ(HoodieTableType tableType, boolean readUtcTimezone) {
TableEnvironment tableEnv = batchTableEnv;
tableEnv.getConfig().setLocalTimeZone(ZoneId.of("America/Los_Angeles"));
String createTable = sql("t1")
@@ -1847,8 +1847,7 @@ public class ITTestHoodieDataSource {
.option(FlinkOptions.PRECOMBINE_FIELD, "f1")
.option(FlinkOptions.TABLE_TYPE, tableType)
.option(FlinkOptions.WRITE_UTC_TIMEZONE, false)
- //FlinkOptions.READ_UTC_TIMEZONE doesn't affect in MergeOnReadInputFormat since the option isn't supported in AvroToRowDataConverters
- //.option(FlinkOptions.READ_UTC_TIMEZONE, false)
+ .option(FlinkOptions.READ_UTC_TIMEZONE, readUtcTimezone)
.pkField("f0")
.noPartition()
.end();
@@ -1870,15 +1869,17 @@ public class ITTestHoodieDataSource {
List<Row> result = CollectionUtil.iterableToList(
() -> tableEnv.sqlQuery("select * from t1").execute().collect());
formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss");
+
+ final ZoneId expectedZoneId = readUtcTimezone ? ZoneId.of("UTC") : ZoneId.systemDefault();
final String expected = "["
+ "+I[1"
+ ", abc"
- + ", " + formatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMillis + 1000), ZoneId.of("UTC")))
- + ", " + formatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMillis + 2000), ZoneId.of("UTC"))) + "], "
+ + ", " + formatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMillis + 1000), expectedZoneId))
+ + ", " + formatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMillis + 2000), expectedZoneId)) + "], "
+ "+I[2"
+ ", def"
- + ", " + formatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMillis + 3000), ZoneId.of("UTC")))
- + ", " + formatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMillis + 4000), ZoneId.of("UTC"))) + "]]";
+ + ", " + formatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMillis + 3000), expectedZoneId))
+ + ", " + formatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMillis + 4000), expectedZoneId)) + "]]";
assertRowsEquals(result, expected);
}
@@ -2013,7 +2014,7 @@ public class ITTestHoodieDataSource {
}
@ParameterizedTest
- @MethodSource("tableTypeAndPartitioningParams")
+ @MethodSource("tableTypeAndBooleanTrueFalseParams")
void testDynamicPartitionPrune(HoodieTableType tableType, boolean hiveStylePartitioning) throws Exception {
Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath());
conf.setString(FlinkOptions.TABLE_NAME, "t1");
@@ -2145,9 +2146,9 @@ public class ITTestHoodieDataSource {
}
/**
- * Return test params => (HoodieTableType, hive style partitioning).
+ * Return test params => (HoodieTableType, true/false).
*/
- private static Stream<Arguments> tableTypeAndPartitioningParams() {
+ private static Stream<Arguments> tableTypeAndBooleanTrueFalseParams() {
Object[][] data =
new Object[][] {
{HoodieTableType.COPY_ON_WRITE, false},