You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by pe...@apache.org on 2022/05/24 14:00:23 UTC
[incubator-linkis] branch dev-1.1.3 updated: [feture-2167] File type variable configuration in FileSource.scala (#2168)
This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.1.3
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.1.3 by this push:
new 8299f98b0 [feture-2167] File type variable configuration in FileSource.scala (#2168)
8299f98b0 is described below
commit 8299f98b0c45368b485461941508098ef62f5726
Author: weixiao <le...@gmail.com>
AuthorDate: Tue May 24 22:00:17 2022 +0800
[feture-2167] File type variable configuration in FileSource.scala (#2168)
* [feture-2167] File type variable configuration in FileSource.scala
* [feture-2167] File type variable configuration in FileSource.scala
---
.../linkis/storage/conf/LinkisStorageConf.scala | 23 ++++++++++++++++++++++
.../apache/linkis/storage/source/FileSource.scala | 4 ++--
2 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala
index 87a8fc263..a6eba05b7 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala
@@ -17,10 +17,13 @@
package org.apache.linkis.storage.conf
+import org.apache.commons.lang.StringUtils
import org.apache.linkis.common.conf.{ByteType, CommonVars}
import org.apache.linkis.common.utils.ByteTimeUtils
object LinkisStorageConf {
+ private val CONF_LOCK = new Object()
+
val HDFS_FILE_SYSTEM_REST_ERRS: String =
CommonVars.apply(
"wds.linkis.hdfs.rest.errs",
@@ -31,4 +34,24 @@ object LinkisStorageConf {
val ROW_BYTE_MAX_LEN = ByteTimeUtils.byteStringAsBytes(ROW_BYTE_MAX_LEN_STR)
+ val FILE_TYPE = CommonVars("wds.linkis.storage.file.type", "dolphin,sql,scala,py,hql,python,out,log,text,sh,jdbc,ngql,psql,fql").getValue
+
+ private var fileTypeArr: Array[String] = null
+
+ private def fileTypeArrParser(fileType: String): Array[String] = {
+ if (StringUtils.isBlank(fileType)) Array()
+ else fileType.split(",")
+ }
+
+ def getFileTypeArr: Array[String] = {
+ if (fileTypeArr == null) {
+ CONF_LOCK.synchronized {
+ if (fileTypeArr == null) {
+ fileTypeArr = fileTypeArrParser(FILE_TYPE)
+ }
+ }
+ }
+ fileTypeArr
+ }
+
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala
index 43138af3d..04f2486af 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala
@@ -19,13 +19,13 @@ package org.apache.linkis.storage.source
import java.io.{Closeable, InputStream}
import java.util
-
import org.apache.linkis.common.io._
import org.apache.linkis.storage.exception.StorageErrorException
import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetReader}
import org.apache.linkis.storage.script.ScriptFsReader
import org.apache.linkis.storage.utils.StorageConfiguration
import org.apache.commons.math3.util.Pair
+import org.apache.linkis.storage.conf.LinkisStorageConf
trait FileSource extends Closeable {
@@ -55,7 +55,7 @@ trait FileSource extends Closeable {
object FileSource {
- private val fileType = Array("dolphin", "sql", "scala", "py", "hql", "python", "out", "log", "text", "sh", "jdbc", "ngql", "psql", "fql")
+ private val fileType = LinkisStorageConf.getFileTypeArr
private val suffixPredicate = (path: String, suffix: String) => path.endsWith(s".$suffix")
def isResultSet(path: String): Boolean = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org