You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/12/01 07:03:03 UTC
(incubator-paimon) 29/46: [oss] FileSystem should be cached by authority (#2417)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.6
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
commit 6adcb269f7b8051f0d769efc5033d2db25c113dd
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Wed Nov 29 19:54:48 2023 +0800
[oss] FileSystem should be cached by authority (#2417)
---
.../org/apache/paimon/oss/HadoopCompliantFileIO.java | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java
index d6fef224b..3f40c577a 100644
--- a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java
+++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
/**
* Hadoop {@link FileIO}.
@@ -39,7 +41,7 @@ public abstract class HadoopCompliantFileIO implements FileIO {
private static final long serialVersionUID = 1L;
- protected transient volatile FileSystem fs;
+ protected transient volatile Map<String, FileSystem> fsMap;
@Override
public SeekableInputStream newInputStream(Path path) throws IOException {
@@ -105,13 +107,22 @@ public abstract class HadoopCompliantFileIO implements FileIO {
}
private FileSystem getFileSystem(org.apache.hadoop.fs.Path path) throws IOException {
- if (fs == null) {
+ if (fsMap == null) {
synchronized (this) {
- if (fs == null) {
- fs = createFileSystem(path);
+ if (fsMap == null) {
+ fsMap = new ConcurrentHashMap<>();
}
}
}
+
+ Map<String, FileSystem> map = fsMap;
+
+ String authority = path.toUri().getAuthority();
+ FileSystem fs = map.get(authority);
+ if (fs == null) {
+ fs = createFileSystem(path);
+ map.put(authority, fs);
+ }
return fs;
}