You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/10/09 15:17:09 UTC
(paimon) branch master updated: [hotfix] Minor refactor to LocalOrphanFilesClean
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new f74001561 [hotfix] Minor refactor to LocalOrphanFilesClean
f74001561 is described below
commit f74001561fdc0b12ff1e0cc91a9d72837ae51173
Author: Jingsong <ji...@gmail.com>
AuthorDate: Wed Oct 9 23:16:57 2024 +0800
[hotfix] Minor refactor to LocalOrphanFilesClean
---
.../paimon/operation/LocalOrphanFilesClean.java | 56 +++++++---------------
1 file changed, 18 insertions(+), 38 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/LocalOrphanFilesClean.java b/paimon-core/src/main/java/org/apache/paimon/operation/LocalOrphanFilesClean.java
index 74f9fbe00..f3f58107a 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/LocalOrphanFilesClean.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/LocalOrphanFilesClean.java
@@ -120,7 +120,24 @@ public class LocalOrphanFilesClean extends OrphanFilesClean {
try {
Set<String> manifests = new HashSet<>();
collectWithoutDataFile(branch, usedFiles::add, manifests::add);
- usedFiles.addAll(retryReadingDataFiles(manifestFile, manifests));
+ List<String> dataFiles = new ArrayList<>();
+ for (String manifestName : manifests) {
+ retryReadingFiles(
+ () -> manifestFile.readWithIOException(manifestName),
+ Collections.<ManifestEntry>emptyList())
+ .stream()
+ .map(ManifestEntry::file)
+ .forEach(
+ f -> {
+ if (candidateDeletes.contains(f.fileName())) {
+ dataFiles.add(f.fileName());
+ }
+ f.extraFiles().stream()
+ .filter(candidateDeletes::contains)
+ .forEach(dataFiles::add);
+ });
+ }
+ usedFiles.addAll(dataFiles);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -148,43 +165,6 @@ public class LocalOrphanFilesClean extends OrphanFilesClean {
return result;
}
- private List<String> retryReadingDataFiles(ManifestFile manifestFile, Set<String> manifestNames)
- throws IOException {
- List<String> dataFiles = new ArrayList<>();
- for (String manifestName : manifestNames) {
- retryReadingFiles(
- () -> manifestFile.readWithIOException(manifestName),
- Collections.<ManifestEntry>emptyList())
- .stream()
- .map(ManifestEntry::file)
- .forEach(
- f -> {
- if (candidateDeletes.contains(f.fileName())) {
- dataFiles.add(f.fileName());
- }
- f.extraFiles().stream()
- .filter(candidateDeletes::contains)
- .forEach(dataFiles::add);
- });
- }
- return dataFiles;
- }
-
- public static List<String> showDeletedFiles(List<Path> deleteFiles, int showLimit) {
- int showSize = Math.min(deleteFiles.size(), showLimit);
- List<String> result = new ArrayList<>();
- if (deleteFiles.size() > showSize) {
- result.add(
- String.format(
- "Total %s files, only %s lines are displayed.",
- deleteFiles.size(), showSize));
- }
- for (int i = 0; i < showSize; i++) {
- result.add(deleteFiles.get(i).toUri().getPath());
- }
- return result;
- }
-
public static List<LocalOrphanFilesClean> createOrphanFilesCleans(
Catalog catalog,
String databaseName,