You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/06/29 15:42:09 UTC
(paimon) branch master updated: [core] Adjust FileIO.copyFile to with overwrite
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 088586dec [core] Adjust FileIO.copyFile to with overwrite
088586dec is described below
commit 088586decc1b6ad3aa84376708cc8c973d347831
Author: Jingsong <ji...@gmail.com>
AuthorDate: Sat Jun 29 23:41:36 2024 +0800
[core] Adjust FileIO.copyFile to with overwrite
---
.../src/main/java/org/apache/paimon/fs/FileIO.java | 17 +++++++-------
.../org/apache/paimon/fs/local/LocalFileIO.java | 9 ++++----
.../test/java/org/apache/paimon/fs/FileIOTest.java | 19 ++-------------
.../org/apache/paimon/fs/local/LocalFleIOTest.java | 21 ++---------------
.../paimon/io/KeyValueFileWriterFactory.java | 2 +-
.../org/apache/paimon/utils/BranchManager.java | 27 ++++++++++++++--------
6 files changed, 35 insertions(+), 60 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java
index 150142495..b79133147 100644
--- a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java
+++ b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java
@@ -271,28 +271,27 @@ public interface FileIO extends Serializable {
/**
* Copy content of one file into another.
*
- * @return false if targetPath file exists
+ * @throws IOException Thrown, if the stream could not be opened because of an I/O, or because
+ * target file already exists at that path and the write mode indicates to not overwrite the
+ * file.
*/
- default boolean copyFile(Path sourcePath, Path targetPath) throws IOException {
- if (exists(targetPath)) {
- return false;
- }
+ default void copyFile(Path sourcePath, Path targetPath, boolean overwrite) throws IOException {
try (SeekableInputStream is = newInputStream(sourcePath);
- PositionOutputStream os = newOutputStream(targetPath, false)) {
+ PositionOutputStream os = newOutputStream(targetPath, overwrite)) {
IOUtils.copy(is, os);
}
- return true;
}
/** Copy all files in sourceDirectory to directory targetDirectory. */
- default void copyFiles(Path sourceDirectory, Path targetDirectory) throws IOException {
+ default void copyFiles(Path sourceDirectory, Path targetDirectory, boolean overwrite)
+ throws IOException {
FileStatus[] fileStatuses = listStatus(sourceDirectory);
List<Path> copyFiles =
Arrays.stream(fileStatuses).map(FileStatus::getPath).collect(Collectors.toList());
for (Path file : copyFiles) {
String fileName = file.getName();
Path targetPath = new Path(targetDirectory.toString() + "/" + fileName);
- copyFile(file, targetPath);
+ copyFile(file, targetPath, overwrite);
}
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/local/LocalFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/local/LocalFileIO.java
index 61154ad62..6e1e27fa4 100644
--- a/paimon-common/src/main/java/org/apache/paimon/fs/local/LocalFileIO.java
+++ b/paimon-common/src/main/java/org/apache/paimon/fs/local/LocalFileIO.java
@@ -218,12 +218,11 @@ public class LocalFileIO implements FileIO {
}
@Override
- public boolean copyFile(Path sourcePath, Path targetPath) throws IOException {
- if (exists(targetPath)) {
- return false;
+ public void copyFile(Path sourcePath, Path targetPath, boolean overwrite) throws IOException {
+ if (!overwrite && exists(targetPath)) {
+ return;
}
- Files.copy(toPath(sourcePath), toPath(targetPath), StandardCopyOption.COPY_ATTRIBUTES);
- return true;
+ Files.copy(toPath(sourcePath), toPath(targetPath), StandardCopyOption.REPLACE_EXISTING);
}
private java.nio.file.Path toPath(Path path) {
diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/FileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/FileIOTest.java
index 1617fe90e..eed26d7a8 100644
--- a/paimon-common/src/test/java/org/apache/paimon/fs/FileIOTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/fs/FileIOTest.java
@@ -22,7 +22,6 @@ import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.options.Options;
-import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -79,26 +78,12 @@ public class FileIOTest {
FileIO fileIO = new DummyFileIO();
fileIO.writeFileUtf8(srcFile, "foobar");
- assertThat(fileIO.copyFileUtf8(srcFile, dstFile)).isTrue();
+ fileIO.copyFile(srcFile, dstFile, true);
assertThat(fileIO.readFileUtf8(dstFile)).isEqualTo("foobar");
fileIO.deleteQuietly(dstFile);
- assertThat(fileIO.copyFile(srcFile, dstFile)).isTrue();
+ fileIO.copyFile(srcFile, dstFile, true);
assertThat(fileIO.readFileUtf8(dstFile)).isEqualTo("foobar");
- fileIO.deleteQuietly(dstFile);
-
- fileIO.deleteQuietly(srcFile);
- srcFile = new Path(this.getClass().getClassLoader().getResource("test-data.orc").toURI());
-
- fileIO.copyFileUtf8(srcFile, dstFile);
- assertThat(FileUtils.contentEquals(new File(srcFile.toUri()), new File(dstFile.toUri())))
- .isFalse();
- fileIO.deleteQuietly(dstFile);
-
- fileIO.copyFile(srcFile, dstFile);
- assertThat(FileUtils.contentEquals(new File(srcFile.toUri()), new File(dstFile.toUri())))
- .isTrue();
- fileIO.deleteQuietly(dstFile);
}
public static void testOverwriteFileUtf8(Path file, FileIO fileIO) throws InterruptedException {
diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/local/LocalFleIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/local/LocalFleIOTest.java
index 5fb444546..2892e64ec 100644
--- a/paimon-common/src/test/java/org/apache/paimon/fs/local/LocalFleIOTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/fs/local/LocalFleIOTest.java
@@ -21,12 +21,9 @@ package org.apache.paimon.fs.local;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
-import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
-import java.io.File;
-
import static org.assertj.core.api.Assertions.assertThat;
/** Tests for {@link LocalFileIO}. */
@@ -42,25 +39,11 @@ public class LocalFleIOTest {
FileIO fileIO = new LocalFileIO();
fileIO.writeFileUtf8(srcFile, "foobar");
- assertThat(fileIO.copyFileUtf8(srcFile, dstFile)).isTrue();
+ fileIO.copyFile(srcFile, dstFile, false);
assertThat(fileIO.readFileUtf8(dstFile)).isEqualTo("foobar");
fileIO.deleteQuietly(dstFile);
- assertThat(fileIO.copyFile(srcFile, dstFile)).isTrue();
+ fileIO.copyFile(srcFile, dstFile, false);
assertThat(fileIO.readFileUtf8(dstFile)).isEqualTo("foobar");
- fileIO.deleteQuietly(dstFile);
-
- fileIO.deleteQuietly(srcFile);
- srcFile = new Path(this.getClass().getClassLoader().getResource("test-data.orc").toURI());
-
- fileIO.copyFileUtf8(srcFile, dstFile);
- assertThat(FileUtils.contentEquals(new File(srcFile.toUri()), new File(dstFile.toUri())))
- .isFalse();
- fileIO.deleteQuietly(dstFile);
-
- fileIO.copyFile(srcFile, dstFile);
- assertThat(FileUtils.contentEquals(new File(srcFile.toUri()), new File(dstFile.toUri())))
- .isTrue();
- fileIO.deleteQuietly(dstFile);
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueFileWriterFactory.java b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueFileWriterFactory.java
index 78c37cf45..cef81e428 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueFileWriterFactory.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueFileWriterFactory.java
@@ -128,7 +128,7 @@ public class KeyValueFileWriterFactory {
throws IOException {
Path sourcePath = formatContext.pathFactory(level).toPath(sourceFileName);
Path targetPath = formatContext.pathFactory(level).toPath(targetFileName);
- fileIO.copyFile(sourcePath, targetPath);
+ fileIO.copyFile(sourcePath, targetPath, true);
}
public FileIO getFileIO() {
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java b/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java
index 52f1e7078..a2ae6d6f7 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java
@@ -111,7 +111,8 @@ public class BranchManager {
TableSchema latestSchema = schemaManager.latest().get();
fileIO.copyFile(
schemaManager.toSchemaPath(latestSchema.id()),
- schemaManager.copyWithBranch(branchName).toSchemaPath(latestSchema.id()));
+ schemaManager.copyWithBranch(branchName).toSchemaPath(latestSchema.id()),
+ true);
} catch (IOException e) {
throw new RuntimeException(
String.format(
@@ -140,10 +141,12 @@ public class BranchManager {
// Copy the corresponding snapshot and schema files into the branch directory
fileIO.copyFile(
snapshotManager.snapshotPath(snapshotId),
- snapshotManager.copyWithBranch(branchName).snapshotPath(snapshot.id()));
+ snapshotManager.copyWithBranch(branchName).snapshotPath(snapshot.id()),
+ true);
fileIO.copyFile(
schemaManager.toSchemaPath(snapshot.schemaId()),
- schemaManager.copyWithBranch(branchName).toSchemaPath(snapshot.schemaId()));
+ schemaManager.copyWithBranch(branchName).toSchemaPath(snapshot.schemaId()),
+ true);
} catch (IOException e) {
throw new RuntimeException(
String.format(
@@ -173,13 +176,16 @@ public class BranchManager {
// Copy the corresponding tag, snapshot and schema files into the branch directory
fileIO.copyFile(
tagManager.tagPath(tagName),
- tagManager.copyWithBranch(branchName).tagPath(tagName));
+ tagManager.copyWithBranch(branchName).tagPath(tagName),
+ true);
fileIO.copyFile(
snapshotManager.snapshotPath(snapshot.id()),
- snapshotManager.copyWithBranch(branchName).snapshotPath(snapshot.id()));
+ snapshotManager.copyWithBranch(branchName).snapshotPath(snapshot.id()),
+ true);
fileIO.copyFile(
schemaManager.toSchemaPath(snapshot.schemaId()),
- schemaManager.copyWithBranch(branchName).toSchemaPath(snapshot.schemaId()));
+ schemaManager.copyWithBranch(branchName).toSchemaPath(snapshot.schemaId()),
+ true);
} catch (IOException e) {
throw new RuntimeException(
String.format(
@@ -272,13 +278,16 @@ public class BranchManager {
fileIO.deleteFilesQuietly(deletePaths);
fileIO.copyFiles(
snapshotManager.copyWithBranch(branchName).snapshotDirectory(),
- snapshotManager.snapshotDirectory());
+ snapshotManager.snapshotDirectory(),
+ true);
fileIO.copyFiles(
schemaManager.copyWithBranch(branchName).schemaDirectory(),
- schemaManager.schemaDirectory());
+ schemaManager.schemaDirectory(),
+ true);
fileIO.copyFiles(
tagManager.copyWithBranch(branchName).tagDirectory(),
- tagManager.tagDirectory());
+ tagManager.tagDirectory(),
+ true);
} catch (IOException e) {
throw new RuntimeException(
String.format(