You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2026/04/01 17:49:54 UTC
(doris) 07/12: [refactor](fs-spi) P4.8-G1G2: delete dead DFSFileSystem subclasses and legacy ObjFileSystem
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-fs-spi
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a844c736ccc95aa6b4c455ef470442850d9e0bbe
Author: morningman <yu...@selectdb.com>
AuthorDate: Wed Apr 1 23:24:30 2026 +0800
[refactor](fs-spi) P4.8-G1G2: delete dead DFSFileSystem subclasses and legacy ObjFileSystem
### What problem does this PR solve?
Issue Number: N/A
Problem Summary: As part of the P4.8 legacy class deletion series, removes dead
code that has no production callers:
- OSSHdfsFileSystem, JFSFileSystem, OFSFileSystem: subclasses of DFSFileSystem
with zero production instantiation (StorageTypeMapper which mapped them was
deleted in P4.8-F)
- org.apache.doris.fs.remote.ObjFileSystem: legacy abstract class whose only
subclasses (S3FileSystem, AzureFileSystem) were deleted in P4.8-F
Also removes their entries from GsonUtils reflection array (ClassNotFoundException
was already handled gracefully, but the entries serve no purpose).
Fixes StageUtilTest to import org.apache.doris.filesystem.spi.ObjFileSystem
(the SPI interface used by production StageUtil) instead of the now-deleted
legacy org.apache.doris.fs.remote.ObjFileSystem.
### Release note
None
### Check List (For Author)
- Test: Regression test / Unit Test / Manual test / No need to test (with reason)
- FE build passes
- Behavior changed: No
- Does this need documentation: No
Co-authored-by: Copilot <22...@users.noreply.github.com>
---
.../org/apache/doris/fs/remote/ObjFileSystem.java | 201 ---------------------
.../apache/doris/fs/remote/dfs/JFSFileSystem.java | 27 ---
.../apache/doris/fs/remote/dfs/OFSFileSystem.java | 27 ---
.../doris/fs/remote/dfs/OSSHdfsFileSystem.java | 27 ---
.../org/apache/doris/persist/gson/GsonUtils.java | 3 -
.../apache/doris/cloud/stage/StageUtilTest.java | 2 +-
6 files changed, 1 insertion(+), 286 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/ObjFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/ObjFileSystem.java
deleted file mode 100644
index 89329d78965..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/ObjFileSystem.java
+++ /dev/null
@@ -1,201 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.fs.remote;
-
-import org.apache.doris.backup.Status;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.foundation.fs.FsStorageType;
-import org.apache.doris.fs.obj.ListObjectsResult;
-import org.apache.doris.fs.obj.ObjStorage;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.file.FileVisitOption;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-
-public abstract class ObjFileSystem extends RemoteFileSystem {
- private static final Logger LOG = LogManager.getLogger(ObjFileSystem.class);
-
- protected final ObjStorage<?> objStorage;
-
- public ObjFileSystem(String name, FsStorageType type, ObjStorage<?> objStorage) {
- super(name, type);
- this.objStorage = objStorage;
- }
-
- public ObjStorage<?> getObjStorage() {
- return objStorage;
- }
-
- @Override
- public Status exists(String remotePath) {
- return objStorage.headObject(remotePath);
- }
-
- @Override
- public Status directoryExists(String dir) {
- return listFiles(dir, false, new ArrayList<>());
- }
-
- /**
- * download data from remote file and check data size with expected file size.
- *
- * @param remoteFilePath remote file path
- * @param localFilePath local file path
- * @param fileSize download data size
- * @return
- */
- @Override
- public Status downloadWithFileSize(String remoteFilePath, String localFilePath, long fileSize) {
- long start = System.currentTimeMillis();
- // Write the data to a local file
- File localFile = new File(localFilePath);
- if (localFile.exists()) {
- try {
- Files.walk(Paths.get(localFilePath), FileVisitOption.FOLLOW_LINKS)
- .sorted(Comparator.reverseOrder())
- .map(Path::toFile)
- .forEach(File::delete);
- } catch (IOException e) {
- return new Status(
- Status.ErrCode.COMMON_ERROR, "failed to delete exist local file: " + localFilePath);
- }
- }
- Status st = objStorage.getObject(remoteFilePath, localFile);
- if (st != Status.OK) {
- return st;
- }
- if (localFile.length() == fileSize) {
- LOG.info(
- "finished to get file from {} to {} with size: {}. cost {} ms",
- remoteFilePath,
- localFile.toPath(),
- fileSize,
- (System.currentTimeMillis() - start));
- return Status.OK;
- } else {
- return new Status(Status.ErrCode.COMMON_ERROR, localFile.toString());
- }
- }
-
- @Override
- public Status directUpload(String content, String remoteFile) {
- Status st = objStorage.putObject(remoteFile, new ByteArrayInputStream(content.getBytes()), content.length());
- if (st != Status.OK) {
- return st;
- }
- LOG.info("upload content success.");
- return Status.OK;
- }
-
- @Override
- public Status upload(String localPath, String remotePath) {
- File localFile = new File(localPath);
- Status st = null;
- try {
- st = objStorage.putObject(remotePath, new FileInputStream(localFile), localFile.length());
- } catch (FileNotFoundException e) {
- throw new RuntimeException(e);
- }
- if (st != Status.OK) {
- return st;
- }
- LOG.info("upload file " + localPath + " success.");
- return Status.OK;
- }
-
- @Override
- public Status makeDir(String remotePath) {
- if (!remotePath.endsWith("/")) {
- remotePath += "/";
- }
- Status st = objStorage.putObject(remotePath, new ByteArrayInputStream(new byte[0]), 0);
- if (st != Status.OK) {
- return st;
- }
- LOG.info("makeDir success.");
- return Status.OK;
- }
-
- @Override
- public Status rename(String origFilePath, String destFilePath) {
- Status status = objStorage.copyObject(origFilePath, destFilePath);
- if (status.ok()) {
- return delete(origFilePath);
- } else {
- return status;
- }
- }
-
- public Status copy(String origFilePath, String destFilePath) {
- return objStorage.copyObject(origFilePath, destFilePath);
- }
-
- @Override
- public Status delete(String remotePath) {
- return objStorage.deleteObject(remotePath);
- }
-
- @Override
- public Status deleteDirectory(String absolutePath) {
- return objStorage.deleteObjects(absolutePath);
- }
-
- public org.apache.commons.lang3.tuple.Triple<String, String, String> getStsToken() throws DdlException {
- return objStorage.getStsToken();
- }
-
- public String getPresignedUrl(String objectKey) throws java.io.IOException {
- return objStorage.getPresignedUrl(objectKey);
- }
-
- public ListObjectsResult listObjectsWithPrefix(
- String prefix, String subPrefix, String continuationToken) throws java.io.IOException {
- return objStorage.listObjectsWithPrefix(prefix, subPrefix, continuationToken);
- }
-
- public ListObjectsResult headObjectWithMeta(String prefix, String subKey) throws java.io.IOException {
- return objStorage.headObjectWithMeta(prefix, subKey);
- }
-
- /**
- * Deletes the given object keys from the specified bucket.
- * Keys should be bare paths without any scheme or bucket prefix.
- */
- public void deleteObjectsByKeys(String bucket, List<String> keys) throws DdlException {
- for (String key : keys) {
- String fullPath = "s3://" + bucket + "/" + key;
- Status st = objStorage.deleteObject(fullPath);
- if (!st.ok()) {
- throw new DdlException("Failed to delete key: " + key + ", error: " + st.getErrMsg());
- }
- }
- }
-
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/JFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/JFSFileSystem.java
deleted file mode 100644
index 00cd11f7ee7..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/JFSFileSystem.java
+++ /dev/null
@@ -1,27 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.fs.remote.dfs;
-
-import org.apache.doris.datasource.property.storage.HdfsCompatibleProperties;
-import org.apache.doris.foundation.fs.FsStorageType;
-
-public class JFSFileSystem extends DFSFileSystem {
- public JFSFileSystem(HdfsCompatibleProperties hdfsProperties) {
- super(hdfsProperties, FsStorageType.JFS);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OFSFileSystem.java
deleted file mode 100644
index 54d91bf9e5e..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OFSFileSystem.java
+++ /dev/null
@@ -1,27 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.fs.remote.dfs;
-
-import org.apache.doris.datasource.property.storage.HdfsCompatibleProperties;
-import org.apache.doris.foundation.fs.FsStorageType;
-
-public class OFSFileSystem extends DFSFileSystem {
- public OFSFileSystem(HdfsCompatibleProperties properties) {
- super(properties, FsStorageType.OFS);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OSSHdfsFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OSSHdfsFileSystem.java
deleted file mode 100644
index 4d8f25de0bd..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OSSHdfsFileSystem.java
+++ /dev/null
@@ -1,27 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.fs.remote.dfs;
-
-import org.apache.doris.datasource.property.storage.OSSHdfsProperties;
-import org.apache.doris.foundation.fs.FsStorageType;
-
-public class OSSHdfsFileSystem extends DFSFileSystem {
- public OSSHdfsFileSystem(OSSHdfsProperties properties) {
- super(properties, FsStorageType.HDFS);
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
index cab4ba5540e..8faf29f12d1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
@@ -586,9 +586,6 @@ public class GsonUtils {
String[][] subtypes = {
{"BrokerFileSystem", "org.apache.doris.fs.remote.BrokerFileSystem"},
{"DFSFileSystem", "org.apache.doris.fs.remote.dfs.DFSFileSystem"},
- {"JFSFileSystem", "org.apache.doris.fs.remote.dfs.JFSFileSystem"},
- {"OFSFileSystem", "org.apache.doris.fs.remote.dfs.OFSFileSystem"},
- {"ObjFileSystem", "org.apache.doris.fs.remote.ObjFileSystem"},
{"S3FileSystem", "org.apache.doris.fs.remote.S3FileSystem"},
{"AzureFileSystem", "org.apache.doris.fs.remote.AzureFileSystem"},
};
diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/stage/StageUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/stage/StageUtilTest.java
index 0151eef7db0..cab4e7a51a4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/cloud/stage/StageUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/stage/StageUtilTest.java
@@ -24,9 +24,9 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.filesystem.spi.ObjFileSystem;
import org.apache.doris.fs.obj.ListObjectsResult;
import org.apache.doris.fs.obj.ObjectFile;
-import org.apache.doris.fs.remote.ObjFileSystem;
import org.apache.doris.thrift.TBrokerFileStatus;
import mockit.Mock;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org