You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by ca...@apache.org on 2022/09/02 03:20:34 UTC
[incubator-linkis] branch dev-1.3.1 updated: feat: scala code format alarm clear in linkis-storage (#3188)
This is an automated email from the ASF dual-hosted git repository.
casion pushed a commit to branch dev-1.3.1
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.3.1 by this push:
new 4cdc4395c feat: scala code format alarm clear in linkis-storage (#3188)
4cdc4395c is described below
commit 4cdc4395ca87dfb473d99946131dda86459d60b7
Author: ruY <43...@users.noreply.github.com>
AuthorDate: Fri Sep 2 11:20:26 2022 +0800
feat: scala code format alarm clear in linkis-storage (#3188)
---
.../org/apache/linkis/storage/FSFactory.scala | 3 +-
.../org/apache/linkis/storage/domain/Dolphin.scala | 3 +-
.../org/apache/linkis/storage/io/IOClient.scala | 3 +-
.../storage/io/IOMethodInterceptorCreator.scala | 3 +-
.../resultset/DefaultResultSetFactory.scala | 12 +++--
.../storage/resultset/StorageResultSetReader.scala | 6 ++-
.../resultset/table/TableResultDeserializer.scala | 11 ++--
.../linkis/storage/script/ScriptFsWriter.scala | 6 +--
.../linkis/storage/script/VariableParser.scala | 62 +++++++++++++---------
.../storage/script/parser/CommonScriptParser.scala | 12 +++--
.../script/reader/StorageScriptFsReader.scala | 3 +-
.../script/writer/StorageScriptFsWriter.scala | 4 +-
.../apache/linkis/storage/source/FileSplit.scala | 6 +--
.../linkis/storage/source/TextFileSource.scala | 4 +-
.../linkis/storage/utils/FileSystemUtils.scala | 3 +-
.../linkis/storage/utils/StorageHelper.scala | 31 +++++++----
16 files changed, 102 insertions(+), 70 deletions(-)
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/FSFactory.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/FSFactory.scala
index 16e78309b..c61d33cfe 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/FSFactory.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/FSFactory.scala
@@ -31,8 +31,9 @@ object FSFactory extends Logging {
)
def getBuildFactory(fsName: String): BuildFactory = {
- if (!buildClasses.contains(fsName))
+ if (!buildClasses.contains(fsName)) {
throw new StorageFatalException(50000, s"Unsupported file system type(不支持的文件系统类型)：$fsName")
+ }
buildClasses(fsName)
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/Dolphin.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/Dolphin.scala
index 33d898820..be9656eb1 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/Dolphin.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/Dolphin.scala
@@ -91,8 +91,9 @@ object Dolphin extends Logging {
}
def getType(content: String): String = {
- if (content.length < MAGIC.length || content.substring(0, MAGIC.length) != MAGIC)
+ if (content.length < MAGIC.length || content.substring(0, MAGIC.length) != MAGIC) {
throw new IOException(s"File header type must be dolphin,content:$content is not")
+ }
content.substring(MAGIC.length, MAGIC.length + INT_LEN).toInt.toString
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOClient.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOClient.scala
index 41ab3111a..0457b8a1f 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOClient.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOClient.scala
@@ -48,11 +48,12 @@ object IOClient {
val FAILED = "FAILED"
def getIOClient(): IOClient = {
- if (ioClient == null)
+ if (ioClient == null) {
throw new StorageErrorException(
52004,
"You must register IOClient before you can use proxy mode.(必须先注册IOClient，才能使用代理模式)"
)
+ }
ioClient
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOMethodInterceptorCreator.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOMethodInterceptorCreator.scala
index a69be6bf6..ec298b517 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOMethodInterceptorCreator.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOMethodInterceptorCreator.scala
@@ -40,11 +40,12 @@ object IOMethodInterceptorCreator {
}
def getIOMethodInterceptor(fsName: String): MethodInterceptor = {
- if (interceptorCreator == null)
+ if (interceptorCreator == null) {
throw new StorageErrorException(
52004,
"You must register IOMethodInterceptorCreator before you can use proxy mode.(必须先注册IOMethodInterceptorCreator，才能使用代理模式)"
)
+ }
interceptorCreator.createIOMethodInterceptor(fsName)
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala
index 60a47269c..f9fd3c9dc 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala
@@ -28,6 +28,7 @@ import org.apache.linkis.storage.utils.{StorageConfiguration, StorageUtils}
import org.apache.commons.lang3.StringUtils
import java.util
+import java.util.Locale
class DefaultResultSetFactory extends ResultSetFactory with Logging {
@@ -35,14 +36,15 @@ class DefaultResultSetFactory extends ResultSetFactory with Logging {
StorageUtils.loadClasses(
StorageConfiguration.STORAGE_RESULT_SET_CLASSES.getValue,
StorageConfiguration.STORAGE_RESULT_SET_PACKAGE.getValue,
- t => t.newInstance().resultSetType().toLowerCase
+ t => t.newInstance().resultSetType().toLowerCase(Locale.getDefault)
)
val resultTypes = ResultSetFactory.resultSetType.keys.toArray
override def getResultSetByType(resultSetType: String): ResultSet[_ <: MetaData, _ <: Record] = {
- if (!resultClasses.contains(resultSetType))
+ if (!resultClasses.contains(resultSetType)) {
throw new StorageErrorException(50000, s"Unsupported result type(不支持的结果类型)：$resultSetType")
+ }
resultClasses(resultSetType).newInstance()
}
@@ -74,11 +76,12 @@ class DefaultResultSetFactory extends ResultSetFactory with Logging {
override def getResultSetByPath(fsPath: FsPath, fs: Fs): ResultSet[_ <: MetaData, _ <: Record] = {
val inputStream = fs.read(fsPath)
val resultSetType = Dolphin.getType(inputStream)
- if (StringUtils.isEmpty(resultSetType))
+ if (StringUtils.isEmpty(resultSetType)) {
throw new StorageWarnException(
51000,
s"The file (${fsPath.getPath}) is empty(文件(${fsPath.getPath}) 为空)"
)
+ }
Utils.tryQuietly(inputStream.close())
// Utils.tryQuietly(fs.close())
getResultSetByType(resultSetType)
@@ -94,11 +97,12 @@ class DefaultResultSetFactory extends ResultSetFactory with Logging {
fs.init(new util.HashMap[String, String]())
val inputStream = fs.read(fsPath)
val resultSetType = Dolphin.getType(inputStream)
- if (StringUtils.isEmpty(resultSetType))
+ if (StringUtils.isEmpty(resultSetType)) {
throw new StorageWarnException(
51000,
s"The file (${fsPath.getPath}) is empty(文件(${fsPath.getPath}) 为空)"
)
+ }
Utils.tryQuietly(inputStream.close())
Utils.tryQuietly(fs.close())
getResultSetByType(resultSetType)
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala
index 97075d6bd..3f49faf3e 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala
@@ -51,11 +51,12 @@ class StorageResultSetReader[K <: MetaData, V <: Record](
def init(): Unit = {
val resType = Dolphin.getType(inputStream)
- if (resultSet.resultSetType != resType)
+ if (resultSet.resultSetType != resType) {
throw new IOException(
"File type does not match(文件类型不匹配): " + ResultSetFactory.resultSetType
.getOrElse(resType, "TABLE")
)
+ }
}
/**
@@ -96,10 +97,11 @@ class StorageResultSetReader[K <: MetaData, V <: Record](
@scala.throws[IOException]
override def getRecord: Record = {
if (metaData == null) throw new IOException("Must read metadata first(必须先读取metadata)")
- if (row == null)
+ if (row == null) {
throw new IOException(
"Can't get the value of the field, maybe the IO stream has been read or has been closed!(拿不到字段的值，也许IO流已读取完毕或已被关闭！)"
)
+ }
row
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultDeserializer.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultDeserializer.scala
index db9ebbfb0..07612ef7e 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultDeserializer.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultDeserializer.scala
@@ -33,12 +33,13 @@ class TableResultDeserializer extends ResultDeserializer[TableMetaData, TableRec
val colByteLen = Dolphin.getString(bytes, 0, Dolphin.INT_LEN).toInt
val colString = Dolphin.getString(bytes, Dolphin.INT_LEN, colByteLen)
val colArray =
- if (colString.endsWith(Dolphin.COL_SPLIT))
+ if (colString.endsWith(Dolphin.COL_SPLIT)) {
colString.substring(0, colString.length - 1).split(Dolphin.COL_SPLIT)
- else colString.split(Dolphin.COL_SPLIT)
+ } else colString.split(Dolphin.COL_SPLIT)
var index = Dolphin.INT_LEN + colByteLen
- if (colArray.length % 3 != 0)
+ if (colArray.length % 3 != 0) {
throw new StorageErrorException(52001, "Parsing metadata failed(解析元数据失败)")
+ }
val columns = new ArrayBuffer[Column]()
for (i <- 0 until (colArray.length, 3)) {
var len = colArray(i).toInt
@@ -66,9 +67,9 @@ class TableResultDeserializer extends ResultDeserializer[TableMetaData, TableRec
val colByteLen = Dolphin.getString(bytes, 0, Dolphin.INT_LEN).toInt
val colString = Dolphin.getString(bytes, Dolphin.INT_LEN, colByteLen)
val colArray =
- if (colString.endsWith(Dolphin.COL_SPLIT))
+ if (colString.endsWith(Dolphin.COL_SPLIT)) {
colString.substring(0, colString.length - 1).split(Dolphin.COL_SPLIT)
- else colString.split(Dolphin.COL_SPLIT)
+ } else colString.split(Dolphin.COL_SPLIT)
var index = Dolphin.INT_LEN + colByteLen
val data = colArray.indices.map { i =>
val len = colArray(i).toInt
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsWriter.scala
index eb2dde5ea..2de8687a2 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsWriter.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsWriter.scala
@@ -97,9 +97,9 @@ trait Compaction {
class ScriptMetaData(var variables: Array[Variable]) extends MetaData {
override def cloneMeta(): MetaData = new ScriptMetaData(variables)
- def getMetaData = variables
+ def getMetaData: Array[Variable] = variables
- def setMetaData(variables: Array[Variable]) = {
+ def setMetaData(variables: Array[Variable]): Unit = {
this.variables = variables
}
@@ -107,5 +107,5 @@ class ScriptMetaData(var variables: Array[Variable]) extends MetaData {
class ScriptRecord(line: String) extends LineRecord(line)
-//definition variable; specialConfiguration ;runConfiguration; startUpConfiguration;
+// definition variable; specialConfiguration ;runConfiguration; startUpConfiguration;
case class Variable(sortParent: String, sort: String, key: String, value: String)
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/VariableParser.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/VariableParser.scala
index c71204f54..ca89bd2f8 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/VariableParser.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/VariableParser.scala
@@ -30,25 +30,28 @@ object VariableParser {
val SPECIAL: String = "special"
def getVariables(params: util.Map[String, Object]): Array[Variable] = {
- import scala.collection.JavaConversions._
+ import scala.collection.JavaConverters._
val variables = new ArrayBuffer[Variable]
params
.getOrDefault(VARIABLE, new util.HashMap[String, Object])
.asInstanceOf[util.Map[String, Object]]
+ .asScala
.foreach(f => variables += Variable(VARIABLE, null, f._1, f._2.toString))
params
.getOrDefault(CONFIGURATION, new util.HashMap[String, Object])
.asInstanceOf[util.Map[String, Object]]
+ .asScala
.foreach { f =>
f._2
.asInstanceOf[util.Map[String, Object]]
+ .asScala
.filter(s => !isContextIDINFO(s._1))
.foreach(p =>
p._2 match {
- case e: util.Map[String, Object] => {
- e.filter(s => !isContextIDINFO(s._1))
+ case e: util.Map[String, Object] =>
+ e.asScala
+ .filter(s => !isContextIDINFO(s._1))
.foreach(s => variables += Variable(f._1, p._1, s._1, s._2.toString))
- }
case _ => variables += Variable(CONFIGURATION, f._1, p._1, p._2.toString)
}
)
@@ -62,41 +65,49 @@ object VariableParser {
}
def getMap(variables: Array[Variable]): util.Map[String, Object] = {
- import scala.collection.JavaConversions._
+ import scala.collection.JavaConverters._
val vars = new util.HashMap[String, String]
val confs = new util.HashMap[String, Object]
- variables.filter(_.sort == null).foreach(f => vars += f.key -> f.value)
+ variables.filter(_.sort == null).foreach(f => vars.asScala += f.key -> f.value)
variables.filter(_.sort != null).foreach { f =>
f.sort match {
case STARTUP | RUNTIME | SPECIAL =>
- if (confs.get(f.sort) == null)
- confs += f.sort -> createMap(f)
- else
- confs.get(f.sort).asInstanceOf[util.HashMap[String, Object]] += f.key -> f.value
+ if (confs.get(f.sort) == null) {
+ confs.asScala += f.sort -> createMap(f)
+ } else {
+ confs.get(f.sort).asInstanceOf[util.HashMap[String, Object]].asScala += f.key -> f.value
+ }
case _ =>
if (confs.get(f.sortParent) == null) {
- confs += f.sortParent -> new util.HashMap[String, Object]
+ confs.asScala += f.sortParent -> new util.HashMap[String, Object]
confs
.get(f.sortParent)
- .asInstanceOf[util.HashMap[String, Object]] += f.sort -> createMap(f)
+ .asInstanceOf[util.HashMap[String, Object]]
+ .asScala += f.sort -> createMap(f)
} else {
val subMap = confs.get(f.sortParent).asInstanceOf[util.HashMap[String, Object]]
- if (subMap.get(f.sort) == null) subMap += f.sort -> createMap(f)
- else subMap.get(f.sort).asInstanceOf[util.HashMap[String, Object]] += f.key -> f.value
+ if (subMap.get(f.sort) == null) subMap.asScala += f.sort -> createMap(f)
+ else
+ subMap
+ .get(f.sort)
+ .asInstanceOf[util.HashMap[String, Object]]
+ .asScala += f.key -> f.value
}
}
}
val params = new util.HashMap[String, Object]
- if (vars.size() > 0) params += VARIABLE -> vars
- if (confs.size() > 0) params += CONFIGURATION -> confs
+ if (vars.size() > 0) params.asScala += VARIABLE -> vars
+ if (confs.size() > 0) params.asScala += CONFIGURATION -> confs
params
}
+ import scala.collection.JavaConverters._
+
private def createMap(variable: Variable): util.Map[String, Object] = {
- import scala.collection.JavaConversions._
+
val map = new util.HashMap[String, Object]
- map += variable.key -> variable.value
- }
+ map.asScala += variable.key -> variable.value
+ }.asJava
/* def main(args: Array[String]): Unit = {
val a = Array(
@@ -106,11 +117,11 @@ object VariableParser {
Variable("configuration", "runtime", "a", "b"),
Variable("runtime", "env", "g2", "h3"),
Variable("startup", "hello", "g2", "h3"))
- //val a = Array[Variable]()
- //println(new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").serializeNulls.create.toJson(getMap(a)))
- //val variables: Array[Variable] = getVariables(getMap(a))
- //val variables: Array[Variable] = getVariables(getMap(a))
- //print(variables)
+ // val a = Array[Variable]()
+ // println(new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").serializeNulls.create.toJson(getMap(a)))
+ // val variables: Array[Variable] = getVariables(getMap(a))
+ // val variables: Array[Variable] = getVariables(getMap(a))
+ // print(variables)
/* val writer = ScriptFsWriter.getScriptFsWriter(new FsPath("file:///tmp/hello.py"), "utf-8", new FileOutputStream("E:\\aaa.py"))
writer.addMetaData(new ScriptMetaData(a))
@@ -123,7 +134,6 @@ object VariableParser {
val map = getMap(reader.getMetaData.asInstanceOf[ScriptMetaData].getMetaData)
println(new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").serializeNulls.create.toJson(map))
print(reader.getRecord.asInstanceOf[ScriptRecord].getLine)
-
- }*/
+ } */
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/CommonScriptParser.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/CommonScriptParser.scala
index 6f670c359..711b44315 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/CommonScriptParser.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/CommonScriptParser.scala
@@ -28,21 +28,23 @@ abstract class CommonScriptParser extends Parser {
line match {
case variableReg(key, value) =>
Variable(VariableParser.VARIABLE, null, key.trim, value.trim)
- case _ => {
+ case _ =>
val split = line.split("=")
- if (split.length != 2)
+ if (split.length != 2) {
throw new StorageErrorException(65000, "Invalid custom parameter(不合法的自定义参数)")
+ }
val value = split(1).trim
val subSplit = split(0).split(" ")
- if (subSplit.filter(_ != "").size != 4)
+ if (subSplit.filter(_ != "").size != 4) {
throw new StorageErrorException(65000, "Invalid custom parameter(不合法的自定义参数)")
- if (!subSplit.filter(_ != "")(0).equals(prefixConf))
+ }
+ if (!subSplit.filter(_ != "")(0).equals(prefixConf)) {
throw new StorageErrorException(65000, "Invalid custom parameter(不合法的自定义参数)")
+ }
val sortParent = subSplit.filter(_ != "")(1).trim
val sort = subSplit.filter(_ != "")(2).trim
val key = subSplit.filter(_ != "")(3).trim
Variable(sortParent, sort, key, value)
- }
}
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala
index 741589834..3f0613783 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala
@@ -101,13 +101,12 @@ class StorageScriptFsReader(val path: FsPath, val charset: String, val inputStre
val regex = ("\\s*" + prefix + "\\s*(.+)\\s*" + "=" + "\\s*(.+)\\s*").r
line match {
case regex(_, _) => true
- case _ => {
+ case _ =>
val split: Array[String] = line.split("=")
if (split.size != 2) return false
if (split(0).split(" ").filter(_ != "").size != 4) return false
if (!split(0).split(" ").filter(_ != "")(0).equals(prefixConf)) return false
true
- }
}
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala
index c415753c9..a318c5504 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala
@@ -53,8 +53,8 @@ class StorageScriptFsWriter(
if (outputStream != null) {
IOUtils.writeLines(metadataLine, "\n", outputStream, charset)
} else {
- import scala.collection.JavaConversions._
- metadataLine.foreach(m => stringBuilder.append(s"$m\n"))
+ import scala.collection.JavaConverters._
+ metadataLine.asScala.foreach(m => stringBuilder.append(s"$m\n"))
}
}
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala
index 731b7911a..0c7e3e7a1 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala
@@ -29,7 +29,7 @@ import org.apache.commons.math3.util.Pair
import java.io.Closeable
import java.util
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
class FileSplit(
var fsReader: FsReader[_ <: MetaData, _ <: Record],
@@ -63,7 +63,7 @@ class FileSplit(
}
def addParams(key: String, value: String): Unit = {
- this.params += key -> value
+ this.params.asScala += key -> value
}
def `while`[M](m: MetaData => M, r: Record => Unit): M = {
@@ -133,7 +133,7 @@ class FileSplit(
"comment" -> column.comment,
"dataType" -> column.dataType.typeName
)
- }
+ }.asJava
// 如果不分页,则一直读,如果分页,则 count需要小于count
def ifContinueRead: Boolean = !pageTrigger || count <= end
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/TextFileSource.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/TextFileSource.scala
index 606c57f03..08ce8f51c 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/TextFileSource.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/TextFileSource.scala
@@ -24,7 +24,7 @@ import org.apache.commons.math3.util.Pair
import java.util
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
class TextFileSource(fileSplits: Array[FileSplit]) extends AbstractFileSource(fileSplits) {
@@ -39,7 +39,7 @@ class TextFileSource(fileSplits: Array[FileSplit]) extends AbstractFileSource(fi
val snds: Array[util.ArrayList[Array[String]]] = collects.map(_.getSecond)
snds.foreach { snd =>
val str = new StringBuilder
- snd.foreach {
+ snd.asScala.foreach {
case Array("\n") => str.append("\n")
case Array(y) => str.append(y).append("\n")
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala
index c40c9a497..5252c12e0 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala
@@ -69,8 +69,9 @@ object FileSystemUtils extends Logging {
): Unit = {
if (!fileSystem.exists(filePath)) {
if (!fileSystem.exists(filePath.getParent)) {
- if (!createParentWhenNotExists)
+ if (!createParentWhenNotExists) {
throw new IOException("parent dir " + filePath.getParent.getPath + " dose not exists.")
+ }
mkdirs(fileSystem, filePath.getParent, user)
}
fileSystem.createNewFile(filePath)
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageHelper.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageHelper.scala
index 3daf69430..7e5169fc5 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageHelper.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageHelper.scala
@@ -17,14 +17,19 @@
package org.apache.linkis.storage.utils
+import org.apache.linkis.common.utils.Logging
import org.apache.linkis.storage.FSFactory
import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetReader}
import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord}
-object StorageHelper {
+/**
+ * 工具类,用于做storage jar包打出来做测试用 Tool class, which is used to print the storage jar package for testing
+ */
+
+object StorageHelper extends Logging {
def main(args: Array[String]): Unit = {
- if (args.length < 2) println("Usage method params eg:getTableResLines path")
+ if (args.length < 2) logger.info("Usage method params eg:getTableResLines path")
val method = args(0)
val params = args.slice(1, args.length)
Thread.sleep(10000L)
@@ -33,7 +38,7 @@ object StorageHelper {
case "getTableResLines" => getTableResLines(params)
case "getTableRes" => getTableRes(params)
case "createNewFile" => createNewFile(params)
- case _ => println("There is no such method")
+ case _ => logger.info("There is no such method")
}
}
@@ -50,14 +55,14 @@ object StorageHelper {
fs.init(null)
val reader = ResultSetReader.getResultSetReader(resultSet, fs.read(resPath))
val rmetaData = reader.getMetaData
- rmetaData.asInstanceOf[TableMetaData].columns.foreach(println)
+ rmetaData.asInstanceOf[TableMetaData].columns.foreach(column => logger.info(column.toString))
var num = 0
Thread.sleep(10000L)
while (reader.hasNext) {
reader.getRecord
num = num + 1
}
- println(num)
+ logger.info(num.toString)
reader.close()
}
@@ -71,8 +76,12 @@ object StorageHelper {
fs.init(null)
val reader = ResultSetReader.getResultSetReader(resultSet, fs.read(resPath))
val rmetaData = reader.getMetaData
- rmetaData.asInstanceOf[TableMetaData].columns.foreach(println)
- rmetaData.asInstanceOf[TableMetaData].columns.map(_.columnName + ",").foreach(print)
+ rmetaData.asInstanceOf[TableMetaData].columns.foreach(column => logger.info(column.toString))
+ rmetaData
+ .asInstanceOf[TableMetaData]
+ .columns
+ .map(_.columnName + ",")
+ .foreach(column => logger.info(column))
var num = 0
while (reader.hasNext) {
num = num + 1
@@ -80,10 +89,10 @@ object StorageHelper {
if (num > len) {
val record = reader.getRecord
record.asInstanceOf[TableRecord].row.foreach { value =>
- print(value.toString)
- print(",")
+ logger.info(value.toString)
+ logger.info(",")
}
- println()
+ logger.info("\n")
}
}
}
@@ -92,7 +101,7 @@ object StorageHelper {
val resPath = StorageUtils.getFsPath(args(0))
val proxyUser = StorageUtils.getJvmUser
FileSystemUtils.createNewFile(resPath, proxyUser, true)
- println("success")
+ logger.info("success")
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org