You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by pe...@apache.org on 2022/05/24 02:27:17 UTC
[incubator-linkis] branch dev-1.1.3 updated: Add support for auto-format in exporting result to excel file. (#2161)
This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.1.3
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.1.3 by this push:
new 3aa1145a5 Add support for auto-format in exporting result to excel file. (#2161)
3aa1145a5 is described below
commit 3aa1145a56fc14ce7eeb2fa0cd4db5af55e3af09
Author: Alexyang <xu...@qq.com>
AuthorDate: Tue May 24 10:27:12 2022 +0800
Add support for auto-format in exporting result to excel file. (#2161)
* 1. engineplugin-pipelne - add param of autoformat on exporting excel
2. storage - add multi cell style support
* 1. storage - change multi cell style support
* 1. computation-engineconn - refactor log when callback entrance is down
* 1. computation-engineconn - refactor log when callback entrance is down
* 1. storage - add multi excel storage
* format code
---
.../apache/linkis/storage/excel/ExcelFsWriter.scala | 3 ++-
.../linkis/storage/excel/StorageExcelWriter.scala | 21 ++++++++++++++++++++-
.../storage/excel/StorageMultiExcelWriter.scala | 4 ++--
.../upstream/handler/ECTaskKillHandler.scala | 5 ++++-
.../pipeline/conf/PipelineEngineConfiguration.scala | 1 +
.../pipeline/executor/ExcelExecutor.scala | 7 ++++---
.../linkis/filesystem/restful/api/FsRestfulApi.java | 14 ++++++++++----
7 files changed, 43 insertions(+), 12 deletions(-)
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelFsWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelFsWriter.scala
index 06b888143..1d4682a28 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelFsWriter.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelFsWriter.scala
@@ -26,9 +26,10 @@ abstract class ExcelFsWriter extends FsWriter {
val charset: String
val sheetName: String
val dateFormat: String
+ val autoFormat: Boolean
}
object ExcelFsWriter {
- def getExcelFsWriter(charset: String, sheetName: String, dateFormat: String, outputStream: OutputStream): ExcelFsWriter = new StorageExcelWriter(charset, sheetName, dateFormat, outputStream: OutputStream)
+ def getExcelFsWriter(charset: String, sheetName: String, dateFormat: String, outputStream: OutputStream, autoFormat: Boolean): ExcelFsWriter = new StorageExcelWriter(charset, sheetName, dateFormat, outputStream: OutputStream, autoFormat: Boolean)
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala
index 4efc0fc72..031ce79ff 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala
@@ -28,9 +28,10 @@ import org.apache.poi.ss.usermodel._
import org.apache.poi.xssf.streaming.{SXSSFSheet, SXSSFWorkbook}
import scala.collection.mutable.ArrayBuffer
+import org.apache.linkis.storage.domain._
-class StorageExcelWriter(val charset: String, val sheetName: String, val dateFormat: String, val outputStream: OutputStream) extends ExcelFsWriter {
+class StorageExcelWriter(val charset: String, val sheetName: String, val dateFormat: String, val outputStream: OutputStream, val autoFormat: Boolean) extends ExcelFsWriter {
protected var workBook: SXSSFWorkbook = _
protected var sheet: SXSSFSheet = _
@@ -73,6 +74,24 @@ class StorageExcelWriter(val charset: String, val sheetName: String, val dateFor
dataType.toString match {
case _ => style.setDataFormat(format.getFormat("@"))
}
+ if (autoFormat) {
+ dataType match {
+ case StringType => style.setDataFormat(format.getFormat("@"))
+ case TinyIntType => style.setDataFormat(format.getFormat("0"))
+ case ShortIntType => style.setDataFormat(format.getFormat("0"))
+ case IntType => style.setDataFormat(format.getFormat("0"))
+ case LongType => style.setDataFormat(format.getFormat("0.00E+00"))
+ case BigIntType => style.setDataFormat(format.getFormat("0.00E+00"))
+ case FloatType => style.setDataFormat(format.getFormat("0.0000000000"))
+ case DoubleType => style.setDataFormat(format.getFormat("0.0000000000"))
+ case CharType => style.setDataFormat(format.getFormat("@"))
+ case VarcharType => style.setDataFormat(format.getFormat("@"))
+ case DateType => style.setDataFormat(format.getFormat("m/d/yy h:mm"))
+ case TimestampType => style.setDataFormat(format.getFormat("m/d/yy h:mm"))
+ case DecimalType => style.setDataFormat(format.getFormat("0.0000000000"))
+ case _ => style.setDataFormat(format.getFormat("@"))
+ }
+ }
style
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageMultiExcelWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageMultiExcelWriter.scala
index 28e6ca8cb..d30e241c4 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageMultiExcelWriter.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageMultiExcelWriter.scala
@@ -21,8 +21,8 @@ import java.io.OutputStream
import org.apache.poi.xssf.streaming.SXSSFWorkbook
-class StorageMultiExcelWriter(override val outputStream: OutputStream)
- extends StorageExcelWriter(null, null, null, outputStream) {
+class StorageMultiExcelWriter(override val outputStream: OutputStream, override val autoFormat: Boolean)
+ extends StorageExcelWriter(null, null, null, outputStream, autoFormat) {
private var sheetIndex = 0
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/handler/ECTaskKillHandler.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/handler/ECTaskKillHandler.scala
index 210eb0478..7767947f7 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/handler/ECTaskKillHandler.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/handler/ECTaskKillHandler.scala
@@ -34,7 +34,10 @@ class ECTaskKillHandler extends MonitorHandler with Logging {
val elements = toBeKilled.iterator
while (elements.hasNext) {
val element = elements.next
- Utils.tryCatch(doKill(element)) {
+ Utils.tryCatch{
+ doKill(element)
+ logger.error(s"ERROR: entrance : ${element.getUpstreamConnection().getUpstreamServiceInstanceName()} lose connect, will kill job : ${element.getKey()}")
+ } {
t => error("Failed to kill job: " + element.getKey, t)
}
}
diff --git a/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/conf/PipelineEngineConfiguration.scala b/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/conf/PipelineEngineConfiguration.scala
index 31fb5eeb5..c07005d51 100644
--- a/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/conf/PipelineEngineConfiguration.scala
+++ b/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/conf/PipelineEngineConfiguration.scala
@@ -30,4 +30,5 @@ object PipelineEngineConfiguration {
val PIPELINE_FIELD_QUOTE_RETOUCH_ENABLE = CommonVars[Boolean](PIPELINE_FIELD_QUOTE_RETOUCH_CONF_ENABLE, false)
+ val EXPORT_EXCEL_AUTO_FORMAT = CommonVars("wds.linkis.pipeline.export.excel.auto_format.enable", false)
}
diff --git a/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/executor/ExcelExecutor.scala b/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/executor/ExcelExecutor.scala
index eb5f8c765..61e351e03 100644
--- a/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/executor/ExcelExecutor.scala
+++ b/linkis-engineconn-plugins/engineconn-plugins/pipeline/src/main/scala/org/apache/linkis/manager/engineplugin/pipeline/executor/ExcelExecutor.scala
@@ -18,7 +18,6 @@
package org.apache.linkis.manager.engineplugin.pipeline.executor
import java.io.OutputStream
-
import org.apache.linkis.common.io.FsPath
import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext
import org.apache.linkis.manager.engineplugin.pipeline.conf.PipelineEngineConfiguration.PIPELINE_OUTPUT_ISOVERWRITE_SWITCH
@@ -30,6 +29,7 @@ import org.apache.linkis.storage.excel.{ExcelFsWriter, StorageMultiExcelWriter}
import org.apache.linkis.storage.fs.FileSystem
import org.apache.linkis.storage.source.FileSource
import org.apache.commons.io.IOUtils
+import org.apache.linkis.manager.engineplugin.pipeline.conf.PipelineEngineConfiguration
class ExcelExecutor extends PipeLineExecutor {
override def execute(sourcePath: String, destPath: String, engineExecutorContext: EngineExecutionContext): ExecuteResponse = {
@@ -42,6 +42,7 @@ class ExcelExecutor extends PipeLineExecutor {
val destFs = FSFactory.getFs(destFsPath)
destFs.init(null)
val outputStream: OutputStream = destFs.write(destFsPath, PIPELINE_OUTPUT_ISOVERWRITE_SWITCH.getValue(options))
+ val excelAutoFormat = PipelineEngineConfiguration.EXPORT_EXCEL_AUTO_FORMAT.getValue(engineExecutorContext.getProperties.asInstanceOf[java.util.Map[String, String]])
if (sourcePath.contains(".")) {
//sourcePaht 是文件形式
// TODO: fs 加目录判断
@@ -49,10 +50,10 @@ class ExcelExecutor extends PipeLineExecutor {
throw new PipeLineErrorException(70003, "Not a result set file(不是结果集文件)")
}
fileSource = FileSource.create(sourceFsPath, sourceFs)
- excelFsWriter = ExcelFsWriter.getExcelFsWriter(DEFAULTC_HARSET, DEFAULT_SHEETNAME, DEFAULT_DATEFORMATE, outputStream)
+ excelFsWriter = ExcelFsWriter.getExcelFsWriter(DEFAULTC_HARSET, DEFAULT_SHEETNAME, DEFAULT_DATEFORMATE, outputStream, excelAutoFormat)
} else {
//目录形式
- excelFsWriter = new StorageMultiExcelWriter(outputStream)
+ excelFsWriter = new StorageMultiExcelWriter(outputStream, excelAutoFormat)
val fsPathListWithError = sourceFs.asInstanceOf[FileSystem].listPathWithError(sourceFsPath)
if (fsPathListWithError == null) {
throw new PipeLineErrorException(70005, "empty dir!")
diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-publicservice/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java
index dee4fff72..adcddf05b 100644
--- a/linkis-public-enhancements/linkis-publicservice/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java
+++ b/linkis-public-enhancements/linkis-publicservice/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java
@@ -531,7 +531,8 @@ public class FsRestfulApi {
String outputFileName,
@RequestParam(value = "sheetName", defaultValue = "result") String sheetName,
@RequestParam(value = "nullValue", defaultValue = "NULL") String nullValue,
- @RequestParam(value = "limit", defaultValue = "0") Integer limit)
+ @RequestParam(value = "limit", defaultValue = "0") Integer limit,
+ @RequestParam(value = "autoFormat", defaultValue = "false") Boolean autoFormat)
throws WorkSpaceException, IOException {
ServletOutputStream outputStream = null;
FsWriter fsWriter = null;
@@ -591,7 +592,11 @@ public class FsRestfulApi {
}
fsWriter =
ExcelFsWriter.getExcelFsWriter(
- charset, sheetName, DEFAULT_DATE_TYPE, outputStream);
+ charset,
+ sheetName,
+ DEFAULT_DATE_TYPE,
+ outputStream,
+ autoFormat);
response.addHeader("Content-Type", XLSX_RESPONSE_CONTENT_TYPE);
if (isLimitDownloadSize) {
fileSource = fileSource.page(1, excelDownloadSize);
@@ -628,7 +633,8 @@ public class FsRestfulApi {
@RequestParam(value = "outputFileName", defaultValue = "downloadResultset")
String outputFileName,
@RequestParam(value = "nullValue", defaultValue = "NULL") String nullValue,
- @RequestParam(value = "limit", defaultValue = "0") Integer limit)
+ @RequestParam(value = "limit", defaultValue = "0") Integer limit,
+ @RequestParam(value = "autoFormat", defaultValue = "false") Boolean autoFormat)
throws WorkSpaceException, IOException {
ServletOutputStream outputStream = null;
FsWriter fsWriter = null;
@@ -672,7 +678,7 @@ public class FsRestfulApi {
if (!FileSource$.MODULE$.isTableResultSet(fileSource)) {
throw WorkspaceExceptionManager.createException(80024);
}
- fsWriter = new StorageMultiExcelWriter(outputStream);
+ fsWriter = new StorageMultiExcelWriter(outputStream, autoFormat);
response.addHeader("Content-Type", XLSX_RESPONSE_CONTENT_TYPE);
if (isLimitDownloadSize) {
fileSource = fileSource.page(1, excelDownloadSize);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org