You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by ca...@apache.org on 2023/03/22 17:25:23 UTC
[linkis] branch dev-1.4.0 updated: Improve exception information and add path information (#4351)
This is an automated email from the ASF dual-hosted git repository.
casion pushed a commit to branch dev-1.4.0
in repository https://gitbox.apache.org/repos/asf/linkis.git
The following commit(s) were added to refs/heads/dev-1.4.0 by this push:
new e24da00b3 Improve exception information and add path information (#4351)
e24da00b3 is described below
commit e24da00b38ba7fd04740f73cef929b16061e8882
Author: binbincheng <10...@users.noreply.github.com>
AuthorDate: Thu Mar 23 01:25:17 2023 +0800
Improve exception information and add path information (#4351)
Co-authored-by: casionone <ca...@gmail.com>
---
.../java/org/apache/linkis/storage/excel/XlsUtils.java | 1 +
.../java/org/apache/linkis/storage/fs/FileSystem.java | 6 ++++++
.../apache/linkis/storage/fs/impl/LocalFileSystem.java | 15 +++++++++++++--
.../storage/resultset/DefaultResultSetFactory.scala | 2 +-
.../apache/linkis/storage/resultset/ResultSetReader.scala | 4 +++-
.../linkis/storage/resultset/StorageResultSet.scala | 1 +
.../linkis/storage/resultset/StorageResultSetWriter.scala | 1 +
7 files changed, 26 insertions(+), 4 deletions(-)
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java
index f3b8f4486..246fb79bc 100644
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java
@@ -61,6 +61,7 @@ public class XlsUtils {
throws Exception {
String hdfsPath =
"/tmp/" + StorageUtils.getJvmUser() + "/" + System.currentTimeMillis() + ".csv";
+ LOG.info("The excel to csv with hdfsPath:" + hdfsPath);
ExcelXlsReader xlsReader = new ExcelXlsReader();
RowToCsvDeal rowToCsvDeal = new RowToCsvDeal();
OutputStream out = null;
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java
index c19213a9c..3067383b6 100644
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java
@@ -24,8 +24,13 @@ import org.apache.linkis.storage.domain.FsPathListWithError;
import java.io.File;
import java.io.IOException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
public abstract class FileSystem implements Fs {
+ private static final Logger LOG = LoggerFactory.getLogger(FileSystem.class);
+
protected String user;
private String defaultFilePerm = "rwxr-----"; // 740
private String defaultFolderPerm = "rwxr-x---"; // 750
@@ -94,6 +99,7 @@ public abstract class FileSystem implements Fs {
} else {
parentPath = path.substring(0, path.lastIndexOf("/"));
}
+ LOG.info("Get Parent Path:" + parentPath);
return new FsPath(parentPath);
}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java
index ce2ee43b7..2df547f10 100644
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java
@@ -75,12 +75,14 @@ public class LocalFileSystem extends FileSystem {
@Override
public long getTotalSpace(FsPath dest) throws IOException {
String path = dest.getPath();
+ LOG.info("Get total space with path:" + path);
return new File(path).getTotalSpace();
}
@Override
public long getFreeSpace(FsPath dest) throws IOException {
String path = dest.getPath();
+ LOG.info("Get free space with path:" + path);
return new File(path).getFreeSpace();
}
@@ -117,6 +119,7 @@ public class LocalFileSystem extends FileSystem {
@Override
public boolean setOwner(FsPath dest, String user) throws IOException {
+ LOG.info("Set owner with path:" + dest.getPath() + "and user:" + user);
if (!StorageUtils.isIOProxy()) {
LOG.info("io not proxy, setOwner skip");
return true;
@@ -133,6 +136,7 @@ public class LocalFileSystem extends FileSystem {
@Override
public boolean setGroup(FsPath dest, String group) throws IOException {
+ LOG.info("Set group with path:" + dest.getPath() + "and group:" + user);
if (!StorageUtils.isIOProxy()) {
LOG.info("io not proxy, setGroup skip");
return true;
@@ -155,6 +159,7 @@ public class LocalFileSystem extends FileSystem {
@Override
public boolean mkdirs(FsPath dest) throws IOException {
String path = dest.getPath();
+ LOG.info("Try to mkdirs with path:" + path);
File file = new File(path);
// Create parent directories one by one and set their permissions to rwxrwxrwx.
Stack<File> dirsToMake = new Stack<File>();
@@ -182,6 +187,7 @@ public class LocalFileSystem extends FileSystem {
}
public boolean canMkdir(FsPath destParentDir) throws IOException {
+ LOG.info("Try to check if the directory can be created with path:" + destParentDir.getPath());
if (!StorageUtils.isIOProxy()) {
LOG.debug("io not proxy, not check owner, just check if have write permission ");
return this.canWrite(destParentDir);
@@ -203,6 +209,7 @@ public class LocalFileSystem extends FileSystem {
@Override
public boolean copy(String origin, String dest) throws IOException {
File file = new File(dest);
+ LOG.info("Try to copy file from:" + origin + " to dest:" + dest);
if (!isOwner(file.getParent())) {
throw new IOException("you have on permission to create file " + dest);
}
@@ -225,6 +232,7 @@ public class LocalFileSystem extends FileSystem {
@Override
public boolean setPermission(FsPath dest, String permission) throws IOException {
+ LOG.info("Try to set permission dest with path:" + dest.getPath());
if (!StorageUtils.isIOProxy()) {
LOG.info("io not proxy, setPermission as parent.");
try {
@@ -251,6 +259,7 @@ public class LocalFileSystem extends FileSystem {
public FsPathListWithError listPathWithError(FsPath path) throws IOException {
File file = new File(path.getPath());
File[] files = file.listFiles();
+ LOG.info("Try to list path:" + path.getPath() + " with error msg");
if (files != null) {
List<FsPath> rtn = new ArrayList();
String message = "";
@@ -294,6 +303,7 @@ public class LocalFileSystem extends FileSystem {
String groupInfo;
try {
groupInfo = Utils.exec(new String[] {"id", user});
+ LOG.info("Get groupinfo:" + groupInfo + " with shell command: id " + user);
} catch (RuntimeException e) {
group = user;
return;
@@ -322,7 +332,7 @@ public class LocalFileSystem extends FileSystem {
} else {
fsPath = new FsPath(dest);
}
-
+ LOG.info("Try to get FsPath with path:" + fsPath.getPath());
PosixFileAttributes attr = null;
try {
attr = Files.readAttributes(Paths.get(fsPath.getPath()), PosixFileAttributes.class);
@@ -365,7 +375,7 @@ public class LocalFileSystem extends FileSystem {
@Override
public boolean create(String dest) throws IOException {
-
+ LOG.info("try to create file with path:" + dest);
File file = new File(dest);
if (!isOwner(file.getParent())) {
throw new IOException("you have on permission to create file " + dest);
@@ -391,6 +401,7 @@ public class LocalFileSystem extends FileSystem {
public List<FsPath> list(FsPath path) throws IOException {
File file = new File(path.getPath());
File[] files = file.listFiles();
+ LOG.info("Try to get file list with path:" + path.getPath());
if (files != null) {
List<FsPath> rtn = new ArrayList();
for (File f : files) {
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala
index 38973ae1a..d4836731d 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala
@@ -108,7 +108,7 @@ class DefaultResultSetFactory extends ResultSetFactory with Logging {
if (StringUtils.isEmpty(resultSetType)) {
throw new StorageWarnException(
THE_FILE_IS_EMPTY.getErrorCode,
- s"The file (${fsPath.getPath}) is empty(文件(${fsPath.getPath}) 为空)"
+ MessageFormat.format(THE_FILE_IS_EMPTY.getErrorDesc, fsPath.getPath)
)
}
Utils.tryQuietly(inputStream.close())
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala
index 663e379b5..e61cf36b3 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala
@@ -19,6 +19,7 @@ package org.apache.linkis.storage.resultset
import org.apache.linkis.common.io.{FsPath, MetaData, Record}
import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetReader}
+import org.apache.linkis.common.utils.Logging
import org.apache.linkis.storage.FSFactory
import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED
import org.apache.linkis.storage.exception.StorageErrorException
@@ -26,7 +27,7 @@ import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord, Ta
import java.io.InputStream
-object ResultSetReader {
+object ResultSetReader extends Logging {
def getResultSetReader[K <: MetaData, V <: Record](
resultSet: ResultSet[K, V],
@@ -83,6 +84,7 @@ object ResultSetReader {
)
}
val fs = FSFactory.getFs(resPath)
+ logger.info("Try to init Fs with path:" + resPath.getPath)
fs.init(null)
ResultSetReader.getResultSetReader(resultSet.asInstanceOf[TableResultSet], fs.read(resPath))
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala
index fc303fbb5..7b3aca62d 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala
@@ -34,6 +34,7 @@ abstract class StorageResultSet[K <: MetaData, V <: Record] extends ResultSet[K,
} else {
parentDir.toPath + "/" + fileName + Dolphin.DOLPHIN_FILE_SUFFIX
}
+ logger.info(s"Get result set path:${path}")
new FsPath(path)
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala
index 50bc0d276..de849078b 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala
@@ -86,6 +86,7 @@ class StorageResultSetWriter[K <: MetaData, V <: Record](
WRITER_LOCK_CREATE.synchronized {
if (!fileCreated) {
if (storePath != null && outputStream == null) {
+ logger.info(s"Try to create a new file:${storePath}, with proxy user:${proxyUser}")
fs = FSFactory.getFsByProxyUser(storePath, proxyUser)
fs.init(null)
FileSystemUtils.createNewFile(storePath, proxyUser, true)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org