You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by ca...@apache.org on 2022/03/31 12:45:44 UTC
[incubator-linkis] branch dev-1.1.2 updated: When downloading data in csv format, the field value is in json format, and the column qualifier can be specified (#1840)
This is an automated email from the ASF dual-hosted git repository.
casion pushed a commit to branch dev-1.1.2
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.1.2 by this push:
new 583edba When downloading data in csv format, the field value is in json format, and the column qualifier can be specified (#1840)
583edba is described below
commit 583edba22ec14b67754239128c01dcf1c9fcb541
Author: weixiao <le...@gmail.com>
AuthorDate: Thu Mar 31 20:45:39 2022 +0800
When downloading data in csv format, the field value is in json format, and the column qualifier can be specified (#1840)
When downloading data in csv format, the field value is in json format, and the column qualifier can be specified
---
.../org/apache/linkis/storage/csv/CSVFsWriter.scala | 3 ++-
.../org/apache/linkis/storage/csv/StorageCSVWriter.scala | 16 +++++++++++++---
.../pipeline/conf/PipelineEngineConfiguration.scala | 4 +++-
.../pipeline/constant/PipeLineConstant.scala | 1 +
.../engineplugin/pipeline/executor/CSVExecutor.scala | 5 ++---
.../linkis/filesystem/restful/api/FsRestfulApi.java | 6 +++++-
6 files changed, 26 insertions(+), 9 deletions(-)
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/CSVFsWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/CSVFsWriter.scala
index adfed30..d16368f 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/CSVFsWriter.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/CSVFsWriter.scala
@@ -25,9 +25,10 @@ import org.apache.linkis.common.io.FsWriter
abstract class CSVFsWriter extends FsWriter {
val charset: String
val separator: String
+ val quoteRetouchEnable: Boolean
}
object CSVFsWriter {
- def getCSVFSWriter(charset: String, separator: String, outputStream: OutputStream): CSVFsWriter = new StorageCSVWriter(charset, separator, outputStream)
+ def getCSVFSWriter(charset: String, separator: String, quoteRetouchEnable: Boolean, outputStream: OutputStream): CSVFsWriter = new StorageCSVWriter(charset, separator, quoteRetouchEnable, outputStream)
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala
index 5591409..748d035 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala
@@ -26,7 +26,7 @@ import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord}
import org.apache.commons.io.IOUtils
-class StorageCSVWriter(val charset: String, val separator: String, val outputStream: OutputStream) extends CSVFsWriter with Logging {
+class StorageCSVWriter(val charset: String, val separator: String, val quoteRetouchEnable: Boolean, val outputStream: OutputStream) extends CSVFsWriter with Logging {
private val delimiter = separator match {
case "," => ','
@@ -43,8 +43,18 @@ class StorageCSVWriter(val charset: String, val separator: String, val outputStr
}
private def compact(row: Array[String]): String = {
- val tmp = row.foldLeft("")((l, r) => l + delimiter + r)
- tmp.substring(1, tmp.length) + "\n"
+ val quotationMarks: String = "\""
+ def decorateValue(v: String): String = {
+ if (v == null || "".equals(v.trim)) v
+ else {
+ if (quoteRetouchEnable) {
+ s"$quotationMarks${v.replaceAll(quotationMarks, "")}$quotationMarks"
+ }
+ else v
+ }
+ }
+
+ row.map(x => decorateValue(x)).toList.mkString(delimiter.toString) + "\n"
}
private def write(row: Array[String]) = {
diff --git a/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/conf/PipelineEngineConfiguration.scala b/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/conf/PipelineEngineConfiguration.scala
index e8c4471..31fb5ee 100644
--- a/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/conf/PipelineEngineConfiguration.scala
+++ b/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/conf/PipelineEngineConfiguration.scala
@@ -18,7 +18,7 @@
package org.apache.linkis.manager.engineplugin.pipeline.conf
import org.apache.linkis.common.conf.CommonVars
-import org.apache.linkis.manager.engineplugin.pipeline.constant.PipeLineConstant.{PIPELINE_FIELD_SPLIT, PIPELINE_OUTPUT_CHARSET, PIPELINE_OUTPUT_ISOVERWRITE}
+import org.apache.linkis.manager.engineplugin.pipeline.constant.PipeLineConstant.{PIPELINE_FIELD_QUOTE_RETOUCH_CONF_ENABLE, PIPELINE_FIELD_SPLIT, PIPELINE_OUTPUT_CHARSET, PIPELINE_OUTPUT_ISOVERWRITE}
object PipelineEngineConfiguration {
@@ -28,4 +28,6 @@ object PipelineEngineConfiguration {
val PIPELINE_FIELD_SPLIT_STR = CommonVars(PIPELINE_FIELD_SPLIT, ",")
+ val PIPELINE_FIELD_QUOTE_RETOUCH_ENABLE = CommonVars[Boolean](PIPELINE_FIELD_QUOTE_RETOUCH_CONF_ENABLE, false)
+
}
diff --git a/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/constant/PipeLineConstant.scala b/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/constant/PipeLineConstant.scala
index 8f693eb..b996373 100644
--- a/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/constant/PipeLineConstant.scala
+++ b/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/constant/PipeLineConstant.scala
@@ -25,5 +25,6 @@ object PipeLineConstant {
val PIPELINE_OUTPUT_SHUFFLE_NULL_TYPE = "pipeline.output.shuffle.null.type"
val PIPELINE_OUTPUT_CHARSET = "pipeline.output.charset"
val PIPELINE_FIELD_SPLIT = "pipeline.field.split"
+ val PIPELINE_FIELD_QUOTE_RETOUCH_CONF_ENABLE = "wds.linkis.engine.pipeline.field.quote.retoch.enable"
val BLANK = "BLANK"
}
diff --git a/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/executor/CSVExecutor.scala b/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/executor/CSVExecutor.scala
index ba81490..8e15bca 100644
--- a/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/executor/CSVExecutor.scala
+++ b/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/executor/CSVExecutor.scala
@@ -18,10 +18,9 @@
package org.apache.linkis.manager.engineplugin.pipeline.executor
import java.io.OutputStream
-
import org.apache.linkis.common.io.FsPath
import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext
-import org.apache.linkis.manager.engineplugin.pipeline.conf.PipelineEngineConfiguration.{PIPELINE_FIELD_SPLIT_STR, PIPELINE_OUTPUT_CHARSET_STR, PIPELINE_OUTPUT_ISOVERWRITE_SWITCH}
+import org.apache.linkis.manager.engineplugin.pipeline.conf.PipelineEngineConfiguration.{PIPELINE_FIELD_QUOTE_RETOUCH_ENABLE, PIPELINE_FIELD_SPLIT_STR, PIPELINE_OUTPUT_CHARSET_STR, PIPELINE_OUTPUT_ISOVERWRITE_SWITCH}
import org.apache.linkis.manager.engineplugin.pipeline.constant.PipeLineConstant._
import org.apache.linkis.manager.engineplugin.pipeline.exception.PipeLineErrorException
import org.apache.linkis.scheduler.executer.ExecuteResponse
@@ -54,7 +53,7 @@ class CSVExecutor extends PipeLineExecutor {
if (BLANK.equalsIgnoreCase(nullValue)) nullValue = ""
val outputStream: OutputStream = destFs.write(destFsPath, PIPELINE_OUTPUT_ISOVERWRITE_SWITCH.getValue(options))
OutputStreamCache.osCache.put(engineExecutionContext.getJobId.get, outputStream)
- val cSVFsWriter = CSVFsWriter.getCSVFSWriter(PIPELINE_OUTPUT_CHARSET_STR.getValue(options), PIPELINE_FIELD_SPLIT_STR.getValue(options), outputStream)
+ val cSVFsWriter = CSVFsWriter.getCSVFSWriter(PIPELINE_OUTPUT_CHARSET_STR.getValue(options), PIPELINE_FIELD_SPLIT_STR.getValue(options), PIPELINE_FIELD_QUOTE_RETOUCH_ENABLE.getValue(options), outputStream)
fileSource.addParams("nullValue", nullValue).write(cSVFsWriter)
IOUtils.closeQuietly(cSVFsWriter)
IOUtils.closeQuietly(fileSource)
diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-publicservice/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java
index 15afbcc..dee4fff 100644
--- a/linkis-public-enhancements/linkis-publicservice/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java
+++ b/linkis-public-enhancements/linkis-publicservice/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java
@@ -525,6 +525,8 @@ public class FsRestfulApi {
@RequestParam(value = "charset", defaultValue = "utf-8") String charset,
@RequestParam(value = "outputFileType", defaultValue = "csv") String outputFileType,
@RequestParam(value = "csvSeperator", defaultValue = ",") String csvSeperator,
+ @RequestParam(value = "quoteRetouchEnable", required = false)
+ boolean quoteRetouchEnable,
@RequestParam(value = "outputFileName", defaultValue = "downloadResultset")
String outputFileName,
@RequestParam(value = "sheetName", defaultValue = "result") String sheetName,
@@ -570,7 +572,9 @@ public class FsRestfulApi {
switch (outputFileType) {
case "csv":
if (FileSource$.MODULE$.isTableResultSet(fileSource)) {
- fsWriter = CSVFsWriter.getCSVFSWriter(charset, csvSeperator, outputStream);
+ fsWriter =
+ CSVFsWriter.getCSVFSWriter(
+ charset, csvSeperator, quoteRetouchEnable, outputStream);
} else {
fsWriter =
ScriptFsWriter.getScriptFsWriter(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org