You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by je...@apache.org on 2026/04/28 14:18:53 UTC
(paimon) branch master updated: [core] Replace reflective FileIO lookups with HadoopOptionsProvider interface (#7722)
This is an automated email from the ASF dual-hosted git repository.
jerry-024 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new fee912c5d4 [core] Replace reflective FileIO lookups with HadoopOptionsProvider interface (#7722)
fee912c5d4 is described below
commit fee912c5d4da700c0430d153a81eda60c4a1a91d
Author: Dapeng Sun(孙大鹏) <da...@alibaba-inc.com>
AuthorDate: Tue Apr 28 22:18:45 2026 +0800
[core] Replace reflective FileIO lookups with HadoopOptionsProvider interface (#7722)
---
.../apache/paimon/fs/HadoopOptionsProvider.java | 38 +++++++++++++
.../java/org/apache/paimon/fs/PluginFileIO.java | 7 ++-
.../org/apache/paimon/fs/hadoop/HadoopFileIO.java | 8 ++-
.../java/org/apache/paimon/jindo/JindoFileIO.java | 4 +-
.../main/java/org/apache/paimon/oss/OSSFileIO.java | 8 ++-
.../org/apache/paimon/format/lance/LanceUtils.java | 62 ++--------------------
.../paimon/format/vortex/VortexReaderFactory.java | 4 +-
.../apache/paimon/format/vortex/VortexUtils.java | 54 +++++--------------
.../paimon/format/vortex/VortexWriterFactory.java | 4 +-
9 files changed, 82 insertions(+), 107 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/HadoopOptionsProvider.java b/paimon-common/src/main/java/org/apache/paimon/fs/HadoopOptionsProvider.java
new file mode 100644
index 0000000000..7754f0f844
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/fs/HadoopOptionsProvider.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fs;
+
+import org.apache.paimon.options.Options;
+
+/**
+ * Optional capability for {@link FileIO} implementations that wrap a Hadoop-compatible filesystem
+ * and want to expose their underlying hadoop-style options to native readers/writers (such as Lance
+ * or Vortex) hosted in modules that cannot depend on the FileIO implementation directly.
+ *
+ * <p>Without this interface those modules fall back to reflective {@code Class.forName} + {@code
+ * Method.invoke} lookups, which silently break when method signatures drift. Implementing this
+ * interface gives them a typed entry point.
+ *
+ * <p>{@code opType} is one of {@code "read"} / {@code "write"} / {@code "meta"}. Implementations
+ * that do not vary options by op type may ignore the argument.
+ */
+public interface HadoopOptionsProvider {
+
+ Options hadoopOptions(Path path, String opType);
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java
index 55fdd291e7..2af4919f08 100644
--- a/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java
+++ b/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java
@@ -27,7 +27,7 @@ import java.io.IOException;
* A {@link FileIO} for plugin jar. {@link FileIO} is serializable, so plugin FileIO should be
* transient.
*/
-public abstract class PluginFileIO implements FileIO {
+public abstract class PluginFileIO implements FileIO, HadoopOptionsProvider {
private static final long serialVersionUID = 1L;
@@ -46,6 +46,11 @@ public abstract class PluginFileIO implements FileIO {
return options;
}
+ @Override
+ public Options hadoopOptions(Path path, String opType) {
+ return options;
+ }
+
@Override
public SeekableInputStream newInputStream(Path path) throws IOException {
return wrap(() -> fileIO(path).newInputStream(path));
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java
index 6db49a919b..3ff241d6c8 100644
--- a/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java
+++ b/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java
@@ -22,6 +22,7 @@ import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.FileStatus;
+import org.apache.paimon.fs.HadoopOptionsProvider;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.PositionOutputStream;
import org.apache.paimon.fs.RemoteIterator;
@@ -50,7 +51,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;
/** Hadoop {@link FileIO}. */
-public class HadoopFileIO implements FileIO {
+public class HadoopFileIO implements FileIO, HadoopOptionsProvider {
private static final long serialVersionUID = 1L;
@@ -91,6 +92,11 @@ public class HadoopFileIO implements FileIO {
return new org.apache.paimon.options.Options(hadoopConf.get());
}
+ @Override
+ public org.apache.paimon.options.Options hadoopOptions(Path path, String opType) {
+ return hadoopOptions();
+ }
+
@Override
public SeekableInputStream newInputStream(Path path) throws IOException {
org.apache.hadoop.fs.Path hadoopPath = path(path);
diff --git a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java
index 01761bc117..c5e2d13a77 100644
--- a/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java
+++ b/paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoFileIO.java
@@ -20,6 +20,7 @@ package org.apache.paimon.jindo;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.HadoopOptionsProvider;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.TwoPhaseOutputStream;
import org.apache.paimon.options.Options;
@@ -48,7 +49,7 @@ import java.util.function.Supplier;
import static org.apache.paimon.options.CatalogOptions.FILE_IO_ALLOW_CACHE;
/** Jindo {@link FileIO}. */
-public class JindoFileIO extends HadoopCompliantFileIO {
+public class JindoFileIO extends HadoopCompliantFileIO implements HadoopOptionsProvider {
private static final long serialVersionUID = 2L;
@@ -166,6 +167,7 @@ public class JindoFileIO extends HadoopCompliantFileIO {
* @param opType read/write/meta
* @return
*/
+ @Override
public Options hadoopOptions(Path path, String opType) {
boolean shouldCache = false;
if (opType.equalsIgnoreCase("read")) {
diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java
index bdfa60e4af..85d7e8cf9e 100644
--- a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java
+++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java
@@ -20,6 +20,7 @@ package org.apache.paimon.oss;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.HadoopOptionsProvider;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.TwoPhaseOutputStream;
import org.apache.paimon.options.Options;
@@ -47,7 +48,7 @@ import java.util.function.Supplier;
import static org.apache.paimon.options.CatalogOptions.FILE_IO_ALLOW_CACHE;
/** OSS {@link FileIO}. */
-public class OSSFileIO extends HadoopCompliantFileIO {
+public class OSSFileIO extends HadoopCompliantFileIO implements HadoopOptionsProvider {
private static final long serialVersionUID = 2L;
@@ -131,6 +132,11 @@ public class OSSFileIO extends HadoopCompliantFileIO {
return hadoopOptions;
}
+ @Override
+ public Options hadoopOptions(Path path, String opType) {
+ return hadoopOptions;
+ }
+
@Override
protected AliyunOSSFileSystem createFileSystem(org.apache.hadoop.fs.Path path) {
final String scheme = path.toUri().getScheme();
diff --git a/paimon-lance/src/main/java/org/apache/paimon/format/lance/LanceUtils.java b/paimon-lance/src/main/java/org/apache/paimon/format/lance/LanceUtils.java
index 4edb803195..9d5828a166 100644
--- a/paimon-lance/src/main/java/org/apache/paimon/format/lance/LanceUtils.java
+++ b/paimon-lance/src/main/java/org/apache/paimon/format/lance/LanceUtils.java
@@ -19,9 +19,8 @@
package org.apache.paimon.format.lance;
import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.HadoopOptionsProvider;
import org.apache.paimon.fs.Path;
-import org.apache.paimon.fs.PluginFileIO;
-import org.apache.paimon.fs.hadoop.HadoopFileIO;
import org.apache.paimon.options.Options;
import org.apache.paimon.rest.RESTTokenFileIO;
import org.apache.paimon.utils.Pair;
@@ -34,11 +33,6 @@ import java.util.Map;
/** Utils for lance all. */
public class LanceUtils {
- private static final Class<?> ossFileIOKlass;
- private static final Class<?> pluginFileIOKlass;
- private static final Class<?> jindoFileIOKlass;
- private static final Class<?> hadoopFileIOKlass;
-
// OSS configuration keys
public static final String FS_OSS_ENDPOINT = "fs.oss.endpoint";
public static final String FS_OSS_ACCESS_KEY_ID = "fs.oss.accessKeyId";
@@ -57,37 +51,6 @@ public class LanceUtils {
public static final String STORAGE_OPTION_OSS_SESSION_TOKEN = "oss_session_token";
public static final String STORAGE_OPTION_OSS_ENDPOINT = "oss_endpoint";
- static {
- Class<?> klass;
- try {
- klass = Class.forName("org.apache.paimon.oss.OSSFileIO");
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- klass = null;
- }
- ossFileIOKlass = klass;
-
- try {
- klass = Class.forName("org.apache.paimon.jindo.JindoFileIO");
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- klass = null;
- }
- jindoFileIOKlass = klass;
-
- try {
- klass = Class.forName("org.apache.paimon.fs.PluginFileIO");
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- klass = null;
- }
- pluginFileIOKlass = klass;
-
- try {
- klass = Class.forName("org.apache.paimon.fs.hadoop.HadoopFileIO");
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- klass = null;
- }
- hadoopFileIOKlass = klass;
- }
-
public static Pair<Path, Map<String, String>> toLanceSpecifiedForReader(
FileIO fileIO, Path path) {
return toLanceSpecified(fileIO, path, true);
@@ -113,26 +76,9 @@ public class LanceUtils {
}
Options originOptions;
- if (ossFileIOKlass != null && ossFileIOKlass.isInstance(fileIO)) {
- try {
- originOptions = (Options) ossFileIOKlass.getMethod("hadoopOptions").invoke(fileIO);
- } catch (Exception e) {
- throw new RuntimeException("Failed to invoke hadoopOptions on OSSFileIO", e);
- }
- } else if (jindoFileIOKlass != null && jindoFileIOKlass.isInstance(fileIO)) {
- try {
- originOptions =
- (Options)
- jindoFileIOKlass
- .getMethod("hadoopOptions", Path.class, String.class)
- .invoke(fileIO, path, isRead ? "read" : "write");
- } catch (Exception e) {
- throw new RuntimeException("Failed to invoke hadoopOptions on JindoFileIO", e);
- }
- } else if (pluginFileIOKlass != null && pluginFileIOKlass.isInstance(fileIO)) {
- originOptions = ((PluginFileIO) fileIO).options();
- } else if (hadoopFileIOKlass != null && hadoopFileIOKlass.isInstance(fileIO)) {
- originOptions = ((HadoopFileIO) fileIO).hadoopOptions();
+ if (fileIO instanceof HadoopOptionsProvider) {
+ originOptions =
+ ((HadoopOptionsProvider) fileIO).hadoopOptions(path, isRead ? "read" : "write");
} else {
originOptions = new Options();
}
diff --git a/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexReaderFactory.java b/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexReaderFactory.java
index 79fde5b8f9..6974ac489d 100644
--- a/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexReaderFactory.java
+++ b/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexReaderFactory.java
@@ -35,7 +35,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import static org.apache.paimon.format.vortex.VortexUtils.toVortexSpecified;
+import static org.apache.paimon.format.vortex.VortexUtils.toVortexSpecifiedForReader;
/** A factory to create Vortex Reader. */
public class VortexReaderFactory implements FormatReaderFactory {
@@ -58,7 +58,7 @@ public class VortexReaderFactory implements FormatReaderFactory {
long[] rowIndices = toRowIndices(context.selection());
Expression predicate = VortexPredicateConverter.toVortexExpression(predicates);
Pair<Path, Map<String, String>> vortexSpecified =
- toVortexSpecified(context.fileIO(), context.filePath());
+ toVortexSpecifiedForReader(context.fileIO(), context.filePath());
return new VortexRecordsReader(
vortexSpecified.getLeft(),
dataSchemaRowType,
diff --git a/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexUtils.java b/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexUtils.java
index a228f5d9b9..c704ed73cb 100644
--- a/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexUtils.java
+++ b/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexUtils.java
@@ -19,8 +19,8 @@
package org.apache.paimon.format.vortex;
import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.HadoopOptionsProvider;
import org.apache.paimon.fs.Path;
-import org.apache.paimon.fs.PluginFileIO;
import org.apache.paimon.options.Options;
import org.apache.paimon.rest.RESTTokenFileIO;
import org.apache.paimon.utils.Pair;
@@ -33,35 +33,18 @@ import java.util.Map;
/** Utils for Vortex storage options. */
public class VortexUtils {
- private static final Class<?> ossFileIOKlass;
- private static final Class<?> pluginFileIO;
- private static final Class<?> jindoFileIOKlass;
-
- static {
- Class<?> klass;
- try {
- klass = Class.forName("org.apache.paimon.oss.OSSFileIO");
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- klass = null;
- }
- ossFileIOKlass = klass;
-
- try {
- klass = Class.forName("org.apache.paimon.jindo.JindoFileIO");
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- klass = null;
- }
- jindoFileIOKlass = klass;
+ public static Pair<Path, Map<String, String>> toVortexSpecifiedForReader(
+ FileIO fileIO, Path path) {
+ return toVortexSpecified(fileIO, path, true);
+ }
- try {
- klass = Class.forName("org.apache.paimon.fs.PluginFileIO");
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- klass = null;
- }
- pluginFileIO = klass;
+ public static Pair<Path, Map<String, String>> toVortexSpecifiedForWriter(
+ FileIO fileIO, Path path) {
+ return toVortexSpecified(fileIO, path, false);
}
- public static Pair<Path, Map<String, String>> toVortexSpecified(FileIO fileIO, Path path) {
+ private static Pair<Path, Map<String, String>> toVortexSpecified(
+ FileIO fileIO, Path path, boolean isRead) {
URI uri = path.toUri();
String schema = uri.getScheme();
@@ -74,12 +57,9 @@ public class VortexUtils {
}
Options originOptions;
- if (ossFileIOKlass != null && ossFileIOKlass.isInstance(fileIO)) {
- originOptions = invokeHadoopOptions(fileIO);
- } else if (jindoFileIOKlass != null && jindoFileIOKlass.isInstance(fileIO)) {
- originOptions = invokeHadoopOptions(fileIO);
- } else if (pluginFileIO != null && pluginFileIO.isInstance(fileIO)) {
- originOptions = ((PluginFileIO) fileIO).options();
+ if (fileIO instanceof HadoopOptionsProvider) {
+ originOptions =
+ ((HadoopOptionsProvider) fileIO).hadoopOptions(path, isRead ? "read" : "write");
} else {
originOptions = new Options();
}
@@ -101,12 +81,4 @@ public class VortexUtils {
return Pair.of(converted, storageOptions);
}
-
- private static Options invokeHadoopOptions(Object fileIO) {
- try {
- return (Options) fileIO.getClass().getMethod("hadoopOptions").invoke(fileIO);
- } catch (Exception e) {
- throw new RuntimeException("Failed to invoke hadoopOptions", e);
- }
- }
}
diff --git a/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexWriterFactory.java b/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexWriterFactory.java
index b9fc61324b..f3de8fca74 100644
--- a/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexWriterFactory.java
+++ b/paimon-vortex/paimon-vortex-format/src/main/java/org/apache/paimon/format/vortex/VortexWriterFactory.java
@@ -32,7 +32,7 @@ import java.io.IOException;
import java.util.Map;
import java.util.function.Supplier;
-import static org.apache.paimon.format.vortex.VortexUtils.toVortexSpecified;
+import static org.apache.paimon.format.vortex.VortexUtils.toVortexSpecifiedForWriter;
/** A factory to create Vortex {@link FormatWriter}. */
public class VortexWriterFactory implements FormatWriterFactory, SupportsDirectWrite {
@@ -55,7 +55,7 @@ public class VortexWriterFactory implements FormatWriterFactory, SupportsDirectW
@Override
public FormatWriter create(FileIO fileIO, Path path, String compression) throws IOException {
- Pair<Path, Map<String, String>> vortexSpecified = toVortexSpecified(fileIO, path);
+ Pair<Path, Map<String, String>> vortexSpecified = toVortexSpecifiedForWriter(fileIO, path);
return new VortexRecordsWriter(
rowType,
arrowFormatWriterSupplier.get(),