You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by pe...@apache.org on 2024/11/05 08:36:24 UTC
(linkis) branch master updated: Add a new management console to modify the engine status to UnHealthy and automatically kill the engine after task execution
This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/linkis.git
The following commit(s) were added to refs/heads/master by this push:
new 9036be666b Add a new management console to modify the engine status to UnHealthy and automatically kill the engine after task execution
9036be666b is described below
commit 9036be666bf177d57753e52bbdfa1eefac172407
Author: aiceflower <ki...@sina.com>
AuthorDate: Tue Nov 5 16:36:18 2024 +0800
Add a new management console to modify the engine status to UnHealthy and automatically kill the engine after task execution
The users can specify fixed engine execution
Add a new management console to modify the engine status to UnHealthy and automatically kill the engine after task execution
Optimize engine reuse logic, large resources cannot reuse small resource engines
Optimize the flink log
Linkis storage change java code to scala (The core process has some compatibility issues, and some of them have been reversed.)
---
.../apache/linkis/common/conf/Configuration.scala | 2 +-
.../linkis/hadoop/common/utils/HDFSUtils.scala | 4 +-
.../java/org/apache/linkis/storage/FSFactory.java | 96 ---
.../org/apache/linkis/storage/LineMetaData.java | 44 -
.../java/org/apache/linkis/storage/LineRecord.java | 47 -
.../linkis/storage/conf/LinkisStorageConf.java | 82 --
.../org/apache/linkis/storage/csv/CSVFsWriter.java | 35 -
.../linkis/storage/csv/StorageCSVWriter.java | 146 ----
.../org/apache/linkis/storage/domain/Column.java | 51 --
.../org/apache/linkis/storage/domain/DataType.java | 237 -----
.../org/apache/linkis/storage/domain/Dolphin.java | 141 ---
.../apache/linkis/storage/domain/MethodEntity.java | 132 ---
.../storage/domain/MethodEntitySerializer.java | 84 --
.../errorcode/LinkisStorageErrorCodeSummary.java | 8 +
.../storage/excel/ExcelAnalysisException.java | 34 -
.../apache/linkis/storage/excel/ExcelFsWriter.java | 41 -
.../linkis/storage/excel/ExcelXlsReader.java | 278 ------
.../apache/linkis/storage/excel/FirstRowDeal.java | 55 --
.../apache/linkis/storage/excel/IExcelRowDeal.java | 26 -
.../apache/linkis/storage/excel/RowToCsvDeal.java | 66 --
.../linkis/storage/excel/StorageExcelWriter.java | 310 -------
.../storage/excel/StorageMultiExcelWriter.java | 58 --
.../org/apache/linkis/storage/excel/XlsUtils.java | 51 +-
.../org/apache/linkis/storage/excel/XlsxUtils.java | 55 +-
.../exception/ColLengthExceedException.java | 34 +
.../exception/ColumnIndexExceedException.java | 34 +
.../linkis/storage/exception/StorageErrorCode.java | 2 -
.../storage/factory/impl/BuildHDFSFileSystem.java | 6 +-
.../storage/factory/impl/BuildLocalFileSystem.java | 8 +-
.../storage/factory/impl/BuildOSSSystem.java | 70 --
.../storage/factory/impl/BuildS3FileSystem.java | 62 --
.../org/apache/linkis/storage/fs/FileSystem.java | 6 +-
.../linkis/storage/fs/impl/HDFSFileSystem.java | 98 ++-
.../linkis/storage/fs/impl/LocalFileSystem.java | 43 +-
.../linkis/storage/fs/impl/OSSFileSystem.java | 398 ---------
.../linkis/storage/fs/impl/S3FileSystem.java | 464 ----------
.../org/apache/linkis/storage/io/IOClient.java | 29 -
.../apache/linkis/storage/io/IOClientFactory.java | 57 --
.../storage/io/IOMethodInterceptorCreator.java | 25 -
.../storage/io/IOMethodInterceptorFactory.java | 49 --
.../storage/resultset/DefaultResultSetFactory.java | 198 -----
.../storage/resultset/OrcResultSetReader.java | 212 -----
.../storage/resultset/OrcResultSetWriter.java | 141 ---
.../storage/resultset/ParquetResultSetReader.java | 158 ----
.../storage/resultset/ParquetResultSetWriter.java | 135 ---
.../linkis/storage/resultset/ResultMetaData.java | 22 -
.../linkis/storage/resultset/ResultRecord.java | 22 -
.../linkis/storage/resultset/ResultSetFactory.java | 74 --
.../storage/resultset/ResultSetReaderFactory.java | 133 ---
.../storage/resultset/ResultSetWriterFactory.java | 102 ---
.../linkis/storage/resultset/StorageResultSet.java | 88 --
.../storage/resultset/StorageResultSetReader.java | 175 ----
.../storage/resultset/StorageResultSetWriter.java | 268 ------
.../storage/resultset/html/HtmlResultSet.java | 48 --
.../linkis/storage/resultset/io/IOMetaData.java | 37 -
.../linkis/storage/resultset/io/IORecord.java | 35 -
.../storage/resultset/io/IOResultDeserializer.java | 37 -
.../storage/resultset/io/IOResultSerializer.java | 47 -
.../linkis/storage/resultset/io/IOResultSet.java | 43 -
.../resultset/picture/PictureResultSet.java | 48 --
.../storage/resultset/table/TableMetaData.java | 44 -
.../storage/resultset/table/TableRecord.java | 35 -
.../resultset/table/TableResultDeserializer.java | 100 ---
.../resultset/table/TableResultSerializer.java | 111 ---
.../storage/resultset/table/TableResultSet.java | 44 -
.../resultset/txt/TextResultDeserializer.java | 36 -
.../resultset/txt/TextResultSerializer.java | 50 --
.../storage/resultset/txt/TextResultSet.java | 46 -
.../apache/linkis/storage/script/Compaction.java | 44 -
.../org/apache/linkis/storage/script/Parser.java | 30 -
.../linkis/storage/script/ParserFactory.java | 31 -
.../linkis/storage/script/ScriptFsReader.java | 40 -
.../linkis/storage/script/ScriptFsWriter.java | 37 -
.../linkis/storage/script/ScriptMetaData.java | 41 -
.../apache/linkis/storage/script/ScriptRecord.java | 26 -
.../org/apache/linkis/storage/script/Variable.java | 48 --
.../linkis/storage/script/VariableParser.java | 131 ---
.../script/compaction/CommonScriptCompaction.java | 47 -
.../script/compaction/PYScriptCompaction.java | 45 -
.../script/compaction/QLScriptCompaction.java | 45 -
.../script/compaction/ScalaScriptCompaction.java | 45 -
.../script/compaction/ShellScriptCompaction.java | 45 -
.../storage/script/parser/CommonScriptParser.java | 75 --
.../storage/script/parser/PYScriptParser.java | 45 -
.../storage/script/parser/QLScriptParser.java | 45 -
.../storage/script/parser/ScalaScriptParser.java | 45 -
.../storage/script/parser/ShellScriptParser.java | 45 -
.../script/reader/StorageScriptFsReader.java | 161 ----
.../script/writer/StorageScriptFsWriter.java | 124 ---
.../linkis/storage/source/AbstractFileSource.java | 124 ---
.../apache/linkis/storage/source/FileSource.java | 181 ----
.../apache/linkis/storage/source/FileSplit.java | 324 -------
.../linkis/storage/source/ResultsetFileSource.java | 64 --
.../linkis/storage/source/TextFileSource.java | 71 --
.../linkis/storage/utils/FileSystemUtils.java | 191 -----
.../org/apache/linkis/storage/utils/OrcUtils.java | 152 ----
.../linkis/storage/utils/StorageConfiguration.java | 161 ----
.../apache/linkis/storage/utils/StorageHelper.java | 157 ----
.../apache/linkis/storage/utils/StorageUtils.java | 279 ------
.../org/apache/linkis/storage/FSFactory.scala | 98 +++
.../org/apache/linkis/storage/LineMetaData.scala | 32 +
.../org/apache/linkis/storage/LineRecord.scala | 34 +
.../linkis/storage/conf/LinkisStorageConf.scala | 81 ++
.../apache/linkis/storage/csv/CSVFsReader.scala | 22 +
.../apache/linkis/storage/csv/CSVFsWriter.scala | 48 ++
.../linkis/storage/csv/StorageCSVReader.scala | 20 +
.../linkis/storage/csv/StorageCSVWriter.scala | 117 +++
.../apache/linkis/storage/domain/DataType.scala | 164 ++++
.../org/apache/linkis/storage/domain/Dolphin.scala | 130 +++
.../linkis/storage/domain/MethodEntity.scala | 97 +++
.../linkis/storage/excel/ExcelFsReader.scala | 22 +
.../linkis/storage/excel/ExcelFsWriter.scala | 47 +
.../linkis/storage/excel/ExcelXlsReader.java | 378 ++++++++
.../linkis/storage/excel/StorageExcelReader.scala | 20 +
.../linkis/storage/excel/StorageExcelWriter.scala | 236 +++++
.../storage/excel/StorageMultiExcelWriter.scala | 53 ++
.../org/apache/linkis/storage/io/IOClient.scala | 75 ++
.../storage/io/IOMethodInterceptorCreator.scala | 53 ++
.../resultset/DefaultResultSetFactory.scala | 130 +++
.../linkis/storage/resultset/ResultMetaData.scala | 22 +
.../linkis/storage/resultset/ResultRecord.scala | 22 +
.../storage/resultset/ResultSetFactory.scala | 69 ++
.../linkis/storage/resultset/ResultSetReader.scala | 93 ++
.../linkis/storage/resultset/ResultSetWriter.scala | 74 ++
.../storage/resultset/StorageResultSet.scala | 51 ++
.../storage/resultset/StorageResultSetReader.scala | 174 ++++
.../storage/resultset/StorageResultSetWriter.scala | 230 +++++
.../storage/resultset/html/HtmlResultSet.scala | 34 +
.../linkis/storage/resultset/io/IOMetaData.scala | 25 +
.../linkis/storage/resultset/io/IORecord.scala | 25 +
.../resultset/io/IOResultDeserializer.scala | 36 +
.../storage/resultset/io/IOResultSerializer.scala | 43 +
.../linkis/storage/resultset/io/IOResultSet.scala | 32 +
.../resultset/picture/PictureResultSet.scala | 34 +
.../storage/resultset/table/TableMetaData.scala | 30 +
.../storage/resultset/table/TableRecord.scala | 30 +
.../resultset/table/TableResultDeserializer.scala | 156 ++++
.../resultset/table/TableResultSerializer.scala | 99 +++
.../storage/resultset/table/TableResultSet.scala | 32 +
.../resultset/txt/TextResultDeserializer.scala | 34 +
.../resultset/txt/TextResultSerializer.scala | 46 +
.../storage/resultset/txt/TextResultSet.scala | 33 +
.../linkis/storage/script/ScriptFsReader.scala | 37 +
.../linkis/storage/script/ScriptFsWriter.scala | 115 +++
.../linkis/storage/script/VariableParser.scala | 117 +++
.../script/compaction/CommonScriptCompaction.scala | 34 +
.../script/compaction/PYScriptCompaction.scala | 39 +
.../script/compaction/QLScriptCompaction.scala | 38 +
.../script/compaction/ScalaScriptCompaction.scala | 38 +
.../script/compaction/ShellScriptCompaction.scala | 39 +
.../storage/script/parser/CommonScriptParser.scala | 63 ++
.../storage/script/parser/PYScriptParser.scala | 39 +
.../storage/script/parser/QLScriptParser.scala | 38 +
.../storage/script/parser/ScalaScriptParser.scala | 39 +
.../storage/script/parser/ShellScriptParser.scala | 39 +
.../script/reader/StorageScriptFsReader.scala | 126 +++
.../script/writer/StorageScriptFsWriter.scala | 120 +++
.../linkis/storage/source/AbstractFileSource.scala | 82 ++
.../apache/linkis/storage/source/FileSource.scala | 160 ++++
.../apache/linkis/storage/source/FileSplit.scala | 202 +++++
.../storage/source/ResultsetFileSource.scala | 53 ++
.../linkis/storage/source/TextFileSource.scala | 52 ++
.../linkis/storage/utils/FileSystemUtils.scala | 171 ++++
.../storage/utils/StorageConfiguration.scala | 85 ++
.../linkis/storage/utils/StorageHelper.scala | 107 +++
.../apache/linkis/storage/utils/StorageUtils.scala | 231 +++++
.../apache/linkis/storage/LineMetaDataTest.java | 36 -
.../StorageResultSetWriterFactoryTest.java | 53 --
.../script/writer/StorageScriptFsWriterTest.java | 8 +-
.../storage/source/ResultsetFileSourceTest.java | 50 --
.../src/test/resources/result-read-test.dolphin | 1 -
.../src/test/resources/storage-read-test.dolphin | 1 -
.../linkis/storage/domain/DataTypeTest.scala | 17 +-
.../storage/utils/StorageConfigurationTest.scala | 5 +-
.../linkis/storage/utils/StorageUtilsTest.scala | 32 +
.../src/test/resources/conf/linkis-cli.properties | 2 +-
.../src/test/resources/linkis-cli.properties | 2 +-
.../linkis/ujes/client/JobObserveActionTest.scala | 2 +-
.../governance/common/utils/ECPathUtils.java | 53 --
.../common/conf/GovernaceCommonConf.scala | 98 +++
.../common/conf/GovernanceCommonConf.scala | 93 --
.../common/constant/job/JobRequestConstants.scala | 5 -
.../exception/GovernanceErrorException.scala | 5 +-
.../common/protocol/task/RequestTask.scala | 3 +-
.../protocol/task/ResponseEngineConnPid.scala | 3 +-
.../common/protocol/task/ResponseTaskExecute.scala | 27 -
.../governance/common/utils/ECPathUtils.scala | 66 ++
.../common/utils/EngineConnArguments.scala | 8 +-
.../governance/common/utils/GovernanceUtils.scala | 27 +-
.../common/utils/OnceExecutorContentUtils.scala | 10 +-
.../common/conf/GovernanceCommonConfTest.scala | 4 +-
.../ecm/core/launch/ProcessEngineConnLaunch.scala | 26 +-
.../apache/linkis/ecm/core/utils/PortUtils.scala | 2 +-
.../src/main/assembly/distribution.xml | 3 -
.../apache/linkis/ecm/restful/ECMRestfulApi.java | 2 +-
.../apache/linkis/ecm/scheduled/EcmClearTask.java | 2 +-
.../ecm/server/operator/EngineConnLogOperator.java | 219 -----
.../server/operator/EngineConnYarnLogOperator.java | 149 ----
.../service/impl/DefaultEngineConnKillService.java | 6 +-
.../org/apache/linkis/ecm/utils/ECMCacheUtils.java | 40 +
.../linkis/ecm/server/conf/ECMConfiguration.scala | 17 +-
.../ecm/server/exception/ECMErrorException.scala | 11 +-
.../linkis/ecm/server/hook/JarUDFLoadECMHook.scala | 1 +
.../server/operator/EngineConnLogOperator.scala | 193 +++++
.../operator/EngineConnYarnLogOperator.scala | 194 +++++
.../ecm/server/service/ECMMetricsService.scala | 20 +
.../impl/AbstractEngineConnLaunchService.scala | 7 +-
.../impl/BmlResourceLocalizationService.scala | 2 +
.../service/impl/DefaultECMMetricsService.scala | 25 +
.../service/impl/DefaultECMRegisterService.scala | 22 +-
.../service/impl/DefaultOperateService.scala | 24 +-
.../impl/ProcessEngineConnLaunchService.scala | 41 +-
.../impl/DefaultECMRegisterServiceTest.java | 55 --
.../operator/OperableOnceEngineConnOperator.java | 92 --
.../engineconn/once/executor/OnceExecutor.scala | 8 +-
.../executor/OnceExecutorExecutionContext.scala | 8 +-
.../operator/OperableOnceEngineConnOperator.scala | 72 ++
.../async/AsyncConcurrentComputationExecutor.scala | 66 +-
.../executor/bml/BmlEnginePreExecuteHook.scala | 4 +
.../executor/conf/ComputationExecutorConf.scala | 8 +-
.../computation/executor/cs/CSTableRegister.scala | 2 +-
.../executor/cs/CSTableResultSetWriter.scala | 2 +-
.../executor/execute/ComputationExecutor.scala | 44 +-
.../execute/ConcurrentComputationExecutor.scala | 51 +-
.../executor/execute/EngineExecutionContext.scala | 37 +-
.../executor/hook/ComputationExecutorHook.scala | 7 +
.../computation/executor/hook/InitSQLHook.scala | 3 +-
.../executor/hook/PythonModuleLoad.scala | 161 ++++
.../hook/PythonModuleLoadEngineConnHook.scala | 64 ++
.../executor/hook/PythonSparkEngineHook.scala | 45 +
.../executor/hook/UDFLoadEngineConnHook.scala | 3 +-
.../executor/hook/UseDatabaseEngineHook.scala | 2 +-
.../executor/hook/executor/ExecuteOnceHook.scala | 8 +
.../computation/executor/rs/RsOutputStream.scala | 6 +-
.../service/TaskExecutionServiceImpl.scala | 255 +++---
.../executor/upstream/ECTaskEntranceMonitor.scala | 2 +-
.../SingleThreadUpstreamConnectionMonitor.scala | 21 +
.../service/ECTaskEntranceMonitorService.scala | 3 +-
.../executor/utlis/ComputationEngineConstant.scala | 2 +
.../common/password/CommandPasswordCallback.java | 75 --
.../common/password/StaticPasswordCallback.java | 40 -
.../engineconn/common/conf/EngineConnConf.scala | 4 -
.../common/conf/EngineConnConstant.scala | 2 +-
.../engineconn/core/util/EngineConnUtils.scala | 2 +-
.../acessible/executor/log/AbstractLogCache.java | 38 +
.../acessible/executor/log/MountLogCache.java | 2 +-
.../acessible/executor/log/SendAppender.java | 38 +-
.../acessible/executor/log/TimeLogCache.java | 41 +
.../impl/EngineConnApplicationInfoOperator.java | 57 --
.../execution/AccessibleEngineConnExecution.scala | 2 +-
.../executor/info/NodeHealthyInfoManager.scala | 2 +-
.../acessible/executor/log/LogHelper.scala | 7 +
.../impl/EngineConnApplicationInfoOperator.scala | 52 ++
.../executor/service/DefaultManagerService.scala | 3 +-
.../executor/service/DefaultOperateService.scala | 29 +-
.../service/EngineConnTimedLockService.scala | 12 +-
.../acessible/executor/service/LockService.scala | 7 +-
.../executor/utils/AccessableExecutorUtils.scala | 1 +
.../callback/hook/CallbackEngineConnHook.scala | 25 +-
.../callback/service/EngineConnCallback.scala | 2 +-
.../service/EngineConnIdentifierCallback.scala | 47 +
.../callback/service/EngineConnPidCallback.scala | 46 -
.../callback/service/EngineConnTimedCallback.scala | 20 +
.../executor/ExecutorExecutionContext.scala | 17 +-
.../conf/EngineConnExecutorConfiguration.scala | 3 +
.../errorcode/EngineconnCoreErrorCodeSummary.java | 6 +-
.../exception/EngineConnPluginErrorCode.scala | 26 +
.../exception/EngineConnPluginErrorException.scala | 7 +-
.../common/resource/UserNodeResource.scala | 22 +
.../apache/linkis/entrance/parser/ParserUtils.java | 2 +-
.../entrance/restful/EntranceRestfulApi.java | 2 +-
.../impl/LogPathCreateInterceptor.scala | 6 +-
.../linkis-application-manager/pom.xml | 6 +
.../loader/EngineConnPluginLoaderConf.java | 44 -
.../loaders/CacheablesEngineConnPluginLoader.java | 3 +-
.../loaders/DefaultEngineConnPluginLoader.java | 27 +-
.../BmlEngineConnPluginResourceLoader.java | 4 +-
.../loader/utils/EngineConnPluginUtils.java | 2 +-
.../server/conf/EngineConnPluginConfiguration.java | 42 -
.../interceptor/EngineConnLaunchInterceptor.java | 30 -
.../loader/EngineConnPluginsLoaderFactory.java | 31 -
.../AbstractEngineConnBmlResourceGenerator.java | 134 ---
.../DefaultEngineConnBmlResourceGenerator.java | 114 ---
.../localize/EngineConnBmlResourceGenerator.java | 30 -
.../localize/EngineConnLocalizeResource.java | 30 -
.../localize/EngineConnLocalizeResourceImpl.java | 69 --
.../service/DefaultEngineConnLaunchService.java | 106 ---
.../DefaultEngineConnResourceFactoryService.java | 80 --
.../service/DefaultEngineConnResourceService.java | 351 --------
.../server/service/EngineConnLaunchService.java | 26 -
.../service/EngineConnResourceFactoryService.java | 30 -
.../server/service/EngineConnResourceRequest.java | 53 --
.../server/service/EngineConnResourceService.java | 43 -
.../service/GetEngineConnResourceRequest.java | 25 -
.../RefreshAllEngineConnResourceRequest.java | 28 -
.../service/RefreshEngineConnResourceRequest.java | 25 -
.../service/impl/EnginePluginAdminServiceImpl.java | 18 +-
.../linkis/manager/LinkisManagerApplication.java | 1 +
.../linkis/manager/am/conf/AMConfiguration.java | 134 ++-
.../ApplicationManagerSpringConfiguration.java | 32 -
.../manager/am/conf/ConfigurationMapCache.java | 4 +-
.../DefaultEngineConnConfigurationService.java | 5 +-
.../linkis/manager/am/conf/ManagerMonitorConf.java | 9 +-
.../am/converter/DefaultMetricsConverter.java | 17 +-
.../manager/am/converter/MetricsConverter.java | 55 ++
.../linkis/manager/am/exception/AMErrorCode.java | 3 +-
.../manager/am/exception/AMErrorException.java | 10 +-
.../linkis/manager/am/label/AMLabelChecker.java | 2 -
.../linkis/manager/am/label/AMLabelFilter.java | 4 +-
.../linkis/manager/am/label/LabelChecker.java | 31 +
.../am/label/MultiUserEngineReuseLabelChooser.java | 2 -
.../manager/am/locker/DefaultEngineNodeLocker.java | 10 +-
.../manager/am/manager/DefaultEMNodeManager.java | 10 +-
.../am/manager/DefaultEngineNodeManager.java | 129 +--
.../linkis/manager/am/manager/EMNodeManager.java | 7 +-
.../manager/am/manager/EngineNodeManager.java | 2 +
.../manager/am/pointer/AbstractNodePointer.java | 18 +-
.../manager/am/pointer/DefaultEMNodPointer.java | 42 +-
.../am/pointer/DefaultEngineNodPointer.java | 10 +-
.../am/pointer/DefaultNodePointerBuilder.java | 3 -
.../linkis/manager/am/pointer/EMNodPointer.java | 33 +
.../manager/am/pointer/EngineNodePointer.java | 34 +
.../linkis/manager/am/pointer/NodePointer.java | 64 ++
.../manager/am/pointer/NodePointerBuilder.java | 28 +
.../am/restful/ECResourceInfoRestfulApi.java | 110 ++-
.../linkis/manager/am/restful/EMRestfulApi.java | 170 +++-
.../manager/am/restful/EngineRestfulApi.java | 109 ++-
.../am/selector/DefaultECAvailableRule.java | 43 -
.../manager/am/selector/DefaultNodeSelector.java | 75 --
.../manager/am/selector/ECAvailableRule.java | 33 -
.../linkis/manager/am/selector/NodeSelector.java | 32 -
.../am/selector/rule/AvailableNodeSelectRule.java | 68 --
.../selector/rule/ConcurrencyNodeSelectRule.java | 37 -
.../am/selector/rule/HotspotExclusionRule.java | 62 --
.../am/selector/rule/NewECMStandbyRule.java | 82 --
.../manager/am/selector/rule/NodeSelectRule.java | 25 -
.../am/selector/rule/ResourceNodeSelectRule.java | 124 ---
.../am/selector/rule/ScoreNodeSelectRule.java | 70 --
.../am/selector/rule/TaskInfoNodeSelectRule.java | 73 --
.../manager/am/service/ECResourceInfoService.java | 17 +-
.../linkis/manager/am/service/EMEngineService.java | 40 -
.../linkis/manager/am/service/EngineService.java | 27 -
.../manager/am/service/HeartbeatService.java | 25 -
.../cache/ConfCacheRemoveBroadcastListener.java | 75 --
.../am/service/em/DefaultECMOperateService.java | 37 -
.../am/service/em/DefaultEMEngineService.java | 231 -----
.../am/service/em/DefaultEMInfoService.java | 142 ---
.../am/service/em/DefaultEMRegisterService.java | 124 ---
.../am/service/em/DefaultEMUnregisterService.java | 79 --
.../manager/am/service/em/ECMOperateService.java | 27 -
.../manager/am/service/em/EMInfoService.java | 34 -
.../manager/am/service/em/EMRegisterService.java | 26 -
.../manager/am/service/em/EMUnregisterService.java | 28 -
.../am/service/engine/AbstractEngineService.java | 41 -
.../engine/DefaultEngineAskEngineService.java | 198 -----
.../engine/DefaultEngineConnCanKillService.java | 196 -----
.../service/engine/DefaultEngineCreateService.java | 441 ----------
.../service/engine/DefaultEngineInfoService.java | 145 ----
.../engine/DefaultEngineOperateService.java | 39 -
.../engine/DefaultEngineRecycleService.java | 92 --
.../service/engine/DefaultEngineReuseService.java | 272 ------
.../service/engine/DefaultEngineStopService.java | 316 -------
.../service/engine/DefaultEngineSwitchService.java | 34 -
.../am/service/engine/EngineAskEngineService.java | 26 -
.../service/engine/EngineConnCanKillService.java | 26 -
.../engine/EngineConnStatusCallbackService.java | 25 -
.../am/service/engine/EngineCreateService.java | 29 -
.../am/service/engine/EngineInfoService.java | 52 --
.../am/service/engine/EngineOperateService.java | 28 -
.../am/service/engine/EngineRecycleService.java | 28 -
.../am/service/engine/EngineReuseService.java | 29 -
.../am/service/engine/EngineStopService.java | 61 --
.../am/service/engine/EngineSwitchService.java | 27 -
.../am/service/heartbeat/AMHeartbeatService.java | 86 --
.../DefaultEngineConnStatusCallbackService.java | 39 +-
.../am/service/impl/ECResourceInfoServiceImpl.java | 86 +-
.../am/service/monitor/NodeHeartbeatMonitor.java | 372 --------
.../org/apache/linkis/manager/am/util/EMUtils.java | 123 +++
.../apache/linkis/manager/am/util/LinkisUtils.java | 248 ------
.../apache/linkis/manager/am/utils/AMUtils.java | 347 --------
.../manager/am/utils/DefaultRetryHandler.java | 122 ---
.../linkis/manager/am/utils/RetryHandler.java | 45 -
.../linkis/manager/am/vo/AMEngineNodeVo.java | 7 +-
.../linkis/manager/am/vo/CanCreateECRes.java | 99 +++
.../org/apache/linkis/manager/am/vo/ConfigVo.java | 70 ++
.../org/apache/linkis/manager/am/vo/EMNodeVo.java | 16 +-
.../linkis/manager/label/LabelManagerUtils.java | 37 -
.../manager/label/conf/LabelManagerConf.java | 29 -
.../label/score/DefaultNodeLabelScorer.java | 205 -----
.../label/score/LabelScoreServiceInstance.java | 51 --
.../manager/label/score/NodeLabelScorer.java | 41 -
.../manager/label/service/NodeLabelAddService.java | 28 -
.../label/service/NodeLabelRemoveService.java | 25 -
.../manager/label/service/NodeLabelService.java | 86 --
.../label/service/ResourceLabelService.java | 62 --
.../manager/label/service/UserLabelService.java | 37 -
.../service/impl/DefaultNodeLabelAddService.java | 69 --
.../impl/DefaultNodeLabelRemoveService.java | 82 --
.../service/impl/DefaultNodeLabelService.java | 497 -----------
.../service/impl/DefaultResourceLabelService.java | 161 ----
.../service/impl/DefaultUserLabelService.java | 167 ----
.../linkis/manager/rm/conf/ResourceStatus.java | 4 +-
.../linkis/manager/rm/domain/RMLabelContainer.java | 52 +-
.../manager/rm/entity/LabelResourceMapping.java | 68 --
.../manager/rm/entity/ResourceOperationType.java | 25 -
.../linkis/manager/rm/exception/RMErrorCode.java | 30 +-
.../kubernetes/KubernetesResourceIdentifier.java | 39 -
.../kubernetes/KubernetesResourceRequester.java | 209 -----
.../parser/KubernetesResourceIdentifierParser.java | 38 -
.../service/impl/ExternalResourceServiceImpl.java | 20 +-
.../manager/rm/external/yarn/YarnQueueInfo.java | 86 ++
.../rm/external/yarn/YarnResourceRequester.java | 95 ++-
.../manager/rm/message/RMMessageService.java | 73 --
.../linkis/manager/rm/restful/RMMonitorRest.java | 880 -------------------
.../manager/rm/service/LabelResourceService.java | 51 --
.../manager/rm/service/RequestResourceService.java | 473 ----------
.../manager/rm/service/ResourceLockService.java | 95 ---
.../linkis/manager/rm/service/ResourceManager.java | 96 ---
.../linkis/manager/rm/service/impl/ChangeType.java | 40 -
.../rm/service/impl/DefaultReqResourceService.java | 37 -
.../rm/service/impl/DefaultResourceManager.java | 886 -------------------
.../DriverAndKubernetesReqResourceService.java | 133 ---
.../impl/DriverAndYarnReqResourceService.java | 151 ----
.../rm/service/impl/LabelResourceServiceImpl.java | 102 ---
.../rm/service/impl/ResourceLogService.java | 217 -----
.../rm/service/impl/UserResourceService.java | 128 ---
.../apache/linkis/manager/rm/utils/AlertUtils.java | 20 -
.../apache/linkis/manager/rm/utils/RMUtils.java | 288 -------
.../manager/rm/utils/RequestKerberosUrlUtils.java | 6 +
.../linkis/manager/rm/utils/UserConfiguration.java | 227 -----
.../manager/service/common/label/LabelChecker.java | 31 -
.../manager/service/common/label/LabelFilter.java | 29 -
.../service/common/label/ManagerLabelService.java | 32 -
.../service/common/metrics/MetricsConverter.java | 55 --
.../service/common/pointer/EMNodPointer.java | 33 -
.../service/common/pointer/EngineNodePointer.java | 34 -
.../service/common/pointer/NodePointer.java | 64 --
.../service/common/pointer/NodePointerBuilder.java | 28 -
.../mapper/common/EngineConnBmlResourceMapper.xml | 93 ++
.../common/ExternalResourceProviderDaoImpl.xml | 4 +-
.../mapper/mysql/EngineConnBmlResourceMapper.xml | 93 --
.../postgresql/EngineConnBmlResourceMapper.xml | 93 --
.../loader/EngineConnPluginLoaderConf.scala | 44 +
.../conf/EngineConnPluginConfiguration.scala | 42 +
.../interceptor/EngineConnLaunchInterceptor.scala | 34 +
.../server/loader/EngineConnPluginsLoader.scala | 32 +
.../AbstractEngineConnBmlResourceGenerator.scala | 123 +++
.../DefaultEngineConnBmlResourceGenerator.scala | 109 +++
.../localize/EngineConnBmlResourceGenerator.scala | 44 +
.../service/DefaultEngineConnLaunchService.scala | 90 ++
.../DefaultEngineConnResourceFactoryService.scala | 64 ++
.../service/DefaultEngineConnResourceService.scala | 324 +++++++
.../server/service/EngineConnLaunchService.scala | 31 +
.../service/EngineConnResourceFactoryService.scala | 33 +
.../server/service/EngineConnResourceService.scala | 78 ++
.../linkis/manager/am/hook/AskEngineConnHook.scala | 25 +
.../manager/am/hook/AskEngineConnHookContext.scala | 26 +
.../manager/am/selector/DefaultNodeSelector.scala | 75 ++
.../manager/am/selector/ECAvailableRule.scala | 55 ++
.../linkis/manager/am/selector/NodeSelector.scala | 31 +
.../am/selector/rule/AvailableNodeSelectRule.scala | 60 ++
.../selector/rule/ConcurrencyNodeSelectRule.scala | 35 +
.../am/selector/rule/HotspotExclusionRule.scala | 70 ++
.../am/selector/rule/NewECMStandbyRule.scala | 83 ++
.../manager/am/selector/rule/NodeSelectRule.scala | 26 +
.../am/selector/rule/OverLoadNodeSelectRule.scala | 66 ++
.../am/selector/rule/ResourceNodeSelectRule.scala | 101 +++
.../am/selector/rule/ScoreNodeSelectRule.scala | 55 ++
.../am/selector/rule/TaskInfoNodeSelectRule.scala | 63 ++
.../manager/am/service/EMEngineService.scala | 39 +
.../linkis/manager/am/service/EngineService.scala | 28 +
.../manager/am/service/HeartbeatService.scala | 26 +
.../cache/ConfCacheRemoveBroadcastListener.scala | 70 ++
.../am/service/em/DefaultECMOperateService.scala | 36 +
.../am/service/em/DefaultEMEngineService.scala | 216 +++++
.../am/service/em/DefaultEMInfoService.scala | 310 +++++++
.../am/service/em/DefaultEMRegisterService.scala | 121 +++
.../am/service/em/DefaultEMUnregisterService.scala | 84 ++
.../manager/am/service/em/ECMOperateService.scala | 27 +
.../manager/am/service/em/EMInfoService.scala | 38 +
.../manager/am/service/em/EMRegisterService.scala | 33 +
.../am/service/em/EMUnregisterService.scala | 29 +
.../am/service/engine/AbstractEngineService.scala | 41 +
.../engine/DefaultEngineAskEngineService.scala | 329 +++++++
.../engine/DefaultEngineConnCanKillService.scala | 173 ++++
.../engine/DefaultEngineCreateService.scala | 474 ++++++++++
.../service/engine/DefaultEngineInfoService.scala | 161 ++++
.../engine/DefaultEngineOperateService.scala | 42 +
.../engine/DefaultEngineRecycleService.scala | 82 ++
.../service/engine/DefaultEngineReuseService.scala | 270 ++++++
.../service/engine/DefaultEngineStopService.scala | 343 ++++++++
.../engine/DefaultEngineSwitchService.scala | 32 +
.../am/service/engine/EngineAskEngineService.scala | 40 +
.../service/engine/EngineConnCanKillService.scala | 26 +
.../am/service/engine/EngineCreateService.scala | 30 +
.../am/service/engine/EngineInfoService.scala | 53 ++
.../am/service/engine/EngineKillService.scala | 31 +
.../am/service/engine/EngineOperateService.scala | 33 +
.../am/service/engine/EngineRecycleService.scala | 29 +
.../am/service/engine/EngineReuseService.scala | 30 +
.../am/service/engine/EngineStopService.scala | 75 ++
.../am/service/engine/EngineSwitchService.scala | 29 +
.../am/service/heartbeat/AMHeartbeatService.scala | 112 +++
.../am/service/monitor/NodeHeartbeatMonitor.scala | 493 +++++++++++
.../apache/linkis/manager/am/utils/AMUtils.scala | 310 +++++++
.../linkis/manager/label/LabelManagerUtils.scala | 37 +
.../label/score/DefaultNodeLabelScorer.scala | 182 ++++
.../label/score/LabelScoreServiceInstance.scala | 35 +
.../manager/label/score/NodeLabelScorer.scala | 46 +
.../label/service/NodeLabelAddService.scala | 28 +
.../label/service/NodeLabelRemoveService.scala | 26 +
.../manager/label/service/NodeLabelService.scala | 103 +++
.../label/service/ResourceLabelService.scala | 68 ++
.../manager/label/service/UserLabelService.scala | 38 +
.../service/impl/DefaultNodeLabelAddService.scala | 56 ++
.../impl/DefaultNodeLabelRemoveService.scala | 75 ++
.../service/impl/DefaultNodeLabelService.scala | 549 ++++++++++++
.../service/impl/DefaultResourceLabelService.scala | 161 ++++
.../service/impl/DefaultUserLabelService.scala | 123 +++
.../manager/rm/entity/LabelResourceMap.scala | 52 ++
.../manager/rm/entity/ResourceOperationType.scala | 26 +
.../manager/rm/message/RMMessageService.scala | 68 ++
.../linkis/manager/rm/restful/RMMonitorRest.scala | 843 ++++++++++++++++++
.../manager/rm/service/LabelResourceService.scala | 56 ++
.../rm/service/RequestResourceService.scala | 531 ++++++++++++
.../manager/rm/service/ResourceLockService.scala | 84 ++
.../manager/rm/service/ResourceManager.scala | 126 +++
.../service/impl/DefaultReqResourceService.scala | 28 +
.../rm/service/impl/DefaultResourceManager.scala | 950 +++++++++++++++++++++
.../impl/DriverAndYarnReqResourceService.scala | 187 ++++
.../rm/service/impl/LabelResourceServiceImpl.scala | 106 +++
.../rm/service/impl/ResourceLogService.scala | 234 +++++
.../rm/service/impl/UserResourceService.scala | 116 +++
.../rm/utils/AcrossClusterRulesJudgeUtils.scala | 246 ++++++
.../apache/linkis/manager/rm/utils/RMUtils.scala | 365 ++++++++
.../manager/rm/utils/UserConfiguration.scala | 174 ++++
.../manager/service/common/label/LabelFilter.scala | 30 +
.../service/common/label/ManagerLabelService.scala | 33 +
.../src/test/java/org/apache/linkis/Scan.java | 26 -
.../org/apache/linkis/WebApplicationServer.java | 34 -
.../engineplugin/server/dao/BaseDaoTest.java | 31 -
.../server/dao/EngineConnBmlResourceDaoTest.java | 111 ---
.../am/util/ECResourceInfoLinkisUtilsTest.java | 52 --
.../manager/am/util/ECResourceInfoUtilsTest.java | 52 ++
.../dao/ExternalResourceProviderDaoTest.java | 40 -
.../src/test/resources/application.properties | 39 -
.../src/test/resources/create.sql | 47 -
.../src/test/resources/create_pg.sql | 48 --
.../src/test/resources/linkis.properties | 25 -
.../manager/label/conf/LabelManagerConfTest.scala | 35 +
.../linkis/manager/rm/utils/RMUtilsTest.scala | 43 +
.../manager/label/conf/LabelCommonConfig.java | 6 +-
.../manager/label/entity/engine/RunType.scala | 2 +-
.../linkis/manager/label/utils/LabelUtil.scala | 21 +
.../linkis/manager/label/TestLabelBuilder.java | 5 +-
.../label/entity/engine/EngineTypeLabelTest.java | 65 --
.../manager/common/conf/ManagerCommonConf.java | 35 -
.../manager/common/conf/RMConfiguration.java | 49 --
.../linkis/manager/common/constant/AMConstant.java | 6 +
.../common/entity/metrics/AMNodeMetrics.java | 11 +
.../manager/common/entity/metrics/NodeMetrics.java | 2 +
.../manager/common/entity/node/AMEMNode.java | 11 +
.../manager/common/entity/node/AMEngineNode.java | 12 +
.../linkis/manager/common/entity/node/AMNode.java | 4 +
.../entity/persistence/PersistenceLabel.java | 17 +-
.../entity/persistence/PersistenceNodeMetrics.java | 11 +
.../entity/persistence/PersistencerEcNodeInfo.java | 4 +-
.../common/entity/resource/CPUResource.java | 2 +-
.../common/entity/resource/CommonNodeResource.java | 30 +
.../entity/resource/DriverAndYarnResource.java | 111 +--
.../common/entity/resource/LoadResource.java | 2 +-
.../common/entity/resource/MemoryResource.java | 2 +-
.../common/entity/resource/NodeResource.java | 12 +
.../manager/common/entity/resource/Resource.java | 16 -
.../common/entity/resource/ResourceType.java | 2 +
.../common/entity/resource/SpecialResource.java | 4 +-
.../errorcode/ManagerCommonErrorCodeSummary.java | 3 +-
.../linkis/manager/common/operator/Operator.java | 56 --
.../manager/common/operator/OperatorFactory.java | 33 -
.../common/operator/OperatorFactoryImpl.java | 72 --
.../linkis/manager/common/protocol/EngineLock.java | 22 -
.../manager/common/protocol/OperateRequest.java | 39 -
.../manager/common/protocol/OperateResponse.java | 28 -
.../manager/common/protocol/RequestEngineLock.java | 48 --
.../common/protocol/RequestEngineUnlock.java | 36 -
.../common/protocol/RequestManagerUnlock.java | 59 --
.../common/protocol/ResponseEngineLock.java | 56 --
.../common/protocol/ResponseEngineUnlock.java | 36 -
.../common/protocol/em/ECMOperateRequest.java | 51 --
.../common/protocol/em/ECMOperateResponse.java | 52 --
.../protocol/em/EMResourceRegisterRequest.java | 79 --
.../common/protocol/em/RegisterEMRequest.java | 79 --
.../common/protocol/em/RegisterEMResponse.java | 50 --
.../protocol/engine/EngineAskAsyncResponse.java | 59 --
.../protocol/engine/EngineAsyncResponse.java | 24 -
.../protocol/engine/EngineConnStatusCallback.java | 68 --
.../engine/EngineConnStatusCallbackToAM.java | 69 --
.../common/protocol/engine/EngineCreateError.java | 66 --
.../protocol/engine/EngineCreateSuccess.java | 53 --
.../protocol/engine/EngineOperateRequest.java | 42 -
.../protocol/engine/EngineOperateResponse.java | 53 --
.../common/protocol/label/LabelReportRequest.java | 43 -
.../common/protocol/label/LabelUpdateRequest.java | 36 -
.../common/protocol/node/NodeHeartbeatMsg.java | 7 +-
.../common/protocol/resource/ResourceProtocol.java | 22 -
.../protocol/resource/ResourceUsedProtocol.java | 51 --
.../protocol/resource/ResourceWithStatus.java | 103 ---
.../protocol/resource/ResponseTaskRunningInfo.java | 74 --
.../resource/ResponseTaskYarnResource.java | 41 -
.../linkis/manager/common/utils/ManagerUtils.java | 7 -
.../linkis/manager/common/utils/ResourceUtils.java | 30 +-
.../linkis/manager/rm/AvailableResource.java | 34 -
.../linkis/manager/rm/NotEnoughResource.java | 36 -
.../linkis/manager/rm/RequestResourceAndWait.java | 47 -
.../org/apache/linkis/manager/rm/ResourceInfo.java | 34 -
.../apache/linkis/manager/rm/ResultResource.java | 20 -
.../linkis/manager/common/operator/Operator.scala | 44 +
.../manager/common/operator/OperatorFactory.scala | 64 ++
.../manager/common/protocol/EngineLock.scala | 39 +
.../manager/common/protocol/OperateRequest.scala | 45 +
.../manager/common/protocol/OperateResponse.scala | 25 +
.../common/protocol/em/ECMOperateRequest.scala | 37 +
.../common/protocol/em/ECMOperateResponse.scala | 32 +
.../protocol/em/EMResourceRegisterRequest.scala | 67 ++
.../common/protocol/em/RegisterEMRequest.scala | 67 ++
.../common/protocol/em/RegisterEMResponse.scala | 24 +
.../protocol/engine/EngineAsyncResponse.scala | 38 +
.../protocol/engine/EngineConnStatusCallback.scala | 55 ++
.../protocol/engine/EngineOperateRequest.scala | 31 +
.../protocol/engine/EngineOperateResponse.scala | 32 +
.../common/protocol/label/LabelUpdateRequest.scala | 27 +
.../protocol/resource/ResourceProtocol.scala | 57 ++
.../protocol/resource/ResourceWithStatus.scala | 33 +
.../apache/linkis/manager/rm/ResourceInfo.scala | 24 +
.../apache/linkis/manager/rm/ResultResource.scala | 24 +
.../linkis/manager/dao/ECResourceRecordMapper.java | 3 +-
.../linkis/manager/dao/LockManagerMapper.java | 7 +
.../linkis/manager/dao/NodeManagerMapper.java | 4 +-
.../LinkisManagerPersistenceErrorCodeSummary.java | 4 +-
.../persistence/NodeManagerPersistence.java | 21 +-
.../persistence/NodeMetricManagerPersistence.java | 12 +-
.../persistence/ResourceManagerPersistence.java | 6 +-
.../impl/DefaultLabelManagerPersistence.java | 8 +
.../impl/DefaultLockManagerPersistence.java | 30 +-
.../impl/DefaultNodeManagerPersistence.java | 70 +-
.../impl/DefaultNodeMetricManagerPersistence.java | 27 +-
.../impl/DefaultResourceLabelPersistence.java | 17 +-
.../impl/DefaultResourceManagerPersistence.java | 12 +-
.../manager/util/PersistenceManagerConf.java | 5 -
.../mapper/common/ECResourceRecordMapper.xml | 109 +++
.../resources/mapper/common/LabelManagerMapper.xml | 542 ++++++++++++
.../resources/mapper/common/NodeManagerMapper.xml | 49 +-
.../mapper/common/NodeMetricManagerMapper.xml | 9 +-
.../mapper/common/ResourceManagerMapper.xml | 101 +++
.../mapper/mysql/ECResourceRecordMapper.xml | 129 ---
.../resources/mapper/mysql/LabelManagerMapper.xml | 585 -------------
.../mapper/mysql/ResourceManagerMapper.xml | 148 ----
.../mapper/postgresql/ECResourceRecordMapper.xml | 127 ---
.../mapper/postgresql/LabelManagerMapper.xml | 542 ------------
.../mapper/postgresql/ResourceManagerMapper.xml | 104 ---
.../manager/dao/ECResourceRecordMapperTest.java | 134 ---
.../linkis/manager/dao/LabelManagerMapperTest.java | 293 -------
.../linkis/manager/dao/NodeManagerMapperTest.java | 1 -
.../manager/dao/ResourceManagerMapperTest.java | 45 +-
.../src/test/resources/application.properties | 20 +-
.../src/test/resources/create.sql | 55 +-
.../src/test/resources/create_pg.sql | 189 ----
linkis-dist/bin/install.sh | 24 +-
.../linkis/templates/configmap-init-sql.yaml | 12 +-
.../linkis/templates/configmap-linkis-config.yaml | 15 +-
linkis-dist/package/admin/configuration_helper.sh | 6 +-
.../conf/linkis-cg-linkismanager.properties | 2 +-
.../package/conf/linkis-cli/linkis-cli.properties | 2 +-
linkis-dist/package/conf/linkis.properties | 19 +-
linkis-dist/package/db/linkis_ddl.sql | 398 +++++++--
linkis-dist/package/db/linkis_dml.sql | 162 ++--
linkis-dist/package/db/linkis_dml_pg.sql | 6 +-
linkis-dist/package/db/module/linkis-mg.sql | 6 +-
.../db/upgrade/1.7.0_schema/mysql/linkis_ddl.sql | 104 +++
.../db/upgrade/1.7.0_schema/mysql/linkis_dml.sql | 104 +++
.../doris/executor/DorisEngineConnExecutor.java | 6 +-
...lasticSearchProcessEngineConnLaunchBuilder.java | 2 +-
.../executor/ElasticSearchEngineConnExecutor.java | 6 +-
.../executor/client/ResponseHandler.scala | 11 +-
.../executor/client/impl/ResponseHandlerImpl.scala | 6 +-
.../flink/errorcode/FlinkErrorCodeSummary.java | 132 +++
.../flink/exception/ExecutorInitException.java | 45 +
.../flink/exception/FlinkInitFailedException.java | 36 +
.../flink/exception/JobExecutionException.java | 36 +
.../flink/exception/SqlExecutionException.java | 36 +
.../flink/exception/SqlParseException.java | 36 +
.../flink/executor/FlinkExecutor.scala | 2 +-
.../executor/FlinkManagerConcurrentExecutor.scala | 103 +++
.../flink/operator/KillOperator.scala | 32 +-
.../flink/operator/ListOperator.scala | 17 +-
.../flink/operator/StatusOperator.scala | 22 +-
.../flink/operator/TriggerSavepointOperator.scala | 25 +-
.../clientmanager/FlinkRestClientManager.scala | 21 +-
.../hive/src/main/resources/log4j2.xml | 4 +-
.../hive/creation/HiveEngineConnFactory.scala | 2 +-
.../HiveEngineConcurrentConnExecutor.scala | 4 +-
.../hive/executor/HiveEngineConnExecutor.scala | 11 +-
.../impala/conf/ImpalaConfiguration.scala | 2 +-
.../io_file/src/main/resources/log4j2.xml | 6 +-
.../io/executor/IoEngineConnExecutor.scala | 97 +--
.../engineplugin/io/service/FsProxyService.scala | 18 +-
.../manager/engineplugin/io/utils/IOHelp.scala | 34 +-
.../java/executor/IoEngineConnExecutorTest.java | 77 --
.../src/test/resources/testIoResult.dolphin | 0
.../engineplugin/jdbc/ConnectionManager.java | 6 -
.../jdbc/errorcode/JDBCErrorCodeSummary.java | 4 +-
.../JDBCProcessEngineConnLaunchBuilder.scala | 2 +-
.../engineplugin/jdbc/conf/JDBCConfiguration.scala | 6 +
.../jdbc/executor/JDBCEngineConnExecutor.scala | 24 +-
.../engineplugin/jdbc/executor/JDBCHelper.java | 38 +-
.../jdbc/executor/JDBCMultiDatasourceParser.scala | 22 +-
.../executor/JDBCMultiDatasourceParserTest.scala | 6 +-
.../jdbc/executor/TestJDBCEngineConnExecutor.scala | 7 +-
.../nebula/src/main/assembly/distribution.xml | 2 +-
.../nebula/executor/NebulaEngineConnExecutor.java | 6 +-
.../builder/OpenLooKengProcessECLaunchBuilder.java | 2 +-
.../executor/OpenLooKengEngineConnExecutor.java | 4 +-
.../pipeline/src/main/resources/log4j2.xml | 3 +
.../executor/PipelineEngineConnExecutor.scala | 3 -
.../PrestoProcessEngineConnLaunchBuilder.java | 2 +-
.../presto/executor/PrestoEngineConnExecutor.java | 4 +-
.../factory/TestPrestoEngineConnFactory.java | 4 +
.../python/PythonEngineConnPlugin.java | 80 --
.../python/conf/PythonEngineConfiguration.java | 51 --
.../errorcode/LinkisPythonErrorCodeSummary.java | 2 +-
.../python/exception/ExecuteException.java | 26 -
.../python/exception/PythonExecuteError.java | 26 -
.../PythonSessionStartFailedExeception.java | 27 -
.../python/executor/PythonEngineConnExecutor.java | 189 ----
.../python/hook/PythonVersionEngineHook.java | 78 --
.../PythonProcessEngineConnLaunchBuilder.java | 22 -
.../manager/engineplugin/python/utils/Kind.java | 92 --
.../python/src/main/resources/log4j2.xml | 3 +
.../python/src/main/resources/python/python.py | 4 +-
.../python/PythonEngineConnPlugin.scala | 75 ++
.../python/conf/PythonEngineConfiguration.scala | 49 ++
.../exception/NoSupportEngineException.scala | 26 +
.../exception/PythonSessionNullException.scala | 35 +
.../python/executor/PythonEngineConnExecutor.scala | 172 ++++
.../python/executor/PythonSession.scala | 43 +-
.../python/factory/PythonEngineConnFactory.scala | 2 +-
.../python/hook/PythonVersionEngineHook.scala | 67 ++
.../PythonProcessEngineConnLaunchBuilder.scala | 22 +
.../manager/engineplugin/python/utils/Kind.scala | 104 +++
.../manager/engineplugin/python/utils/State.scala | 74 ++
.../python/TestPythonEngineConnPlugin.java | 30 -
.../python/conf/TestPythonEngineConfiguration.java | 41 -
.../exception/TestNoSupportEngineException.java | 47 -
.../executor/TestPythonEngineConnExecutor.java | 82 --
.../factory/TestPythonEngineConnFactory.java | 44 -
.../engineplugin/python/utils/TestKind.java | 42 -
.../python/TestPythonEngineConnPlugin.scala | 30 +
.../conf/TestPythonEngineConfiguration.scala | 43 +
.../exception/TestNoSupportEngineException.scala | 95 +++
.../executor/TestPythonEngineConnExecutor.scala | 82 ++
.../factory/TestPythonEngineConnFactory.scala | 47 +
.../engineplugin/python/utils/TestKind.scala | 43 +
.../engineplugin/python/utils/TestState.scala | 37 +
.../repl/executor/ReplEngineConnExecutor.java | 6 +-
.../engineplugin/shell/ShellEngineConnPlugin.java | 81 --
.../ShellProcessEngineConnLaunchBuilder.java | 22 -
.../shell/common/ShellEngineConnPluginConst.java | 24 -
.../shell/conf/ShellEngineConnConf.java | 30 -
.../shell/exception/NoCorrectUserException.java | 28 -
.../shell/exception/ShellCodeErrorException.java | 28 -
.../engineplugin/shell/executor/ReaderThread.java | 97 ---
.../shell/executor/ShellECTaskInfo.java | 54 --
.../ShellEngineConnConcurrentExecutor.java | 151 ----
.../shell/executor/ShellEngineConnExecutor.java | 339 --------
.../shell/executor/YarnAppIdExtractor.java | 53 --
.../resources/conf/linkis-engineconn.properties | 3 +-
.../shell/src/main/resources/conf/log4j2.xml | 5 +-
.../engineplugin/shell/ShellEngineConnPlugin.scala | 76 ++
.../ShellProcessEngineConnLaunchBuilder.scala | 22 +
.../shell/common/ShellEnginePluginConst.scala | 23 +
.../shell/conf/ShellEngineConnConf.scala | 30 +
.../shell/exception/NoCorrectUserException.scala | 27 +
.../engineplugin/shell/executor/ReaderThread.scala | 105 +++
.../shell/executor/ShellECTaskInfo.scala | 20 +
.../ShellEngineConnConcurrentExecutor.scala | 378 ++++++++
.../shell/executor/ShellEngineConnExecutor.scala | 337 ++++++++
.../shell/executor/YarnAppIdExtractor.scala | 81 ++
.../shell/factory/ShellEngineConnFactory.scala | 17 +-
.../shell/TestShellEngineConnPlugin.java | 33 -
.../common/TestShellEngineConnPluginConst.java | 32 -
.../exception/TestNoCorrectUserException.java | 34 -
.../executor/TestShellEngineConnExecutor.java | 62 --
.../shell/TestShellEngineConnPlugin.scala | 33 +
.../common/TestShellEngineConnPluginConst.scala | 33 +
.../exception/TestNoCorrectUserException.scala | 38 +
.../executor/TestShellEngineConnExecutor.scala | 62 ++
.../spark/datacalc/source/SolrSourceConfig.java | 45 -
.../spark/datacalc/util/PluginUtil.java | 1 -
.../main/resources/linkis-engineconn.properties | 2 +-
.../spark/src/main/resources/python/mix_pyspark.py | 31 +-
.../engineplugin/spark/common/SparkKind.scala | 22 -
.../spark/config/SparkConfiguration.scala | 5 +-
.../spark/datacalc/source/SolrSource.scala | 44 -
.../engineplugin/spark/executor/SQLSession.scala | 4 +-
.../spark/executor/SparkEngineConnExecutor.scala | 42 +-
.../spark/executor/SparkPythonExecutor.scala | 8 +-
.../spark/executor/SparkScalaExecutor.scala | 4 +-
.../spark/executor/SparkSubmitOnceExecutor.scala | 2 +-
.../spark/factory/SparkEngineConnFactory.scala | 4 +
.../engineplugin/spark/imexport/ExportData.scala | 13 +-
.../engineplugin/spark/imexport/LoadData.scala | 57 +-
...SparkSubmitProcessEngineConnLaunchBuilder.scala | 18 +-
.../spark/mdq/MDQPostExecutionHook.scala | 14 +-
.../spark/mdq/MDQPreExecutionHook.scala | 17 +-
.../engineplugin/spark/utils/EngineUtils.scala | 4 +-
.../execution/datasources/csv/DolphinToSpark.scala | 26 +-
.../spark/src/test/resources/etltest.dolphin | 4 -
.../engineplugin/spark/cs/TestCSSparkHelper.scala | 5 +
.../spark/cs/TestCSSparkPostExecutionHook.scala | 5 +
.../spark/datacalc/TestExcelCala.scala | 135 ---
.../spark/datacalc/TestKafkaCala.scala | 118 ---
.../spark/datacalc/TestRedisCalc.scala | 200 -----
.../spark/datacalc/TestRocketmqCala.scala | 71 --
.../spark/executor/TestSparkSqlExecutor.scala | 72 +-
.../sqoop/src/main/assembly/distribution.xml | 4 -
.../sqoop/client/config/ParamsMapping.java | 6 +
.../sqoop/src/main/resources/log4j2.xml | 18 +-
.../sqoop/client/RemoteClientHolder.scala | 97 +++
.../exception/DataSourceRpcErrorException.scala | 33 +
.../sqoop/executor/SqoopOnceCodeExecutor.scala | 10 +-
.../sqoop/params/ConnectParamsResolver.scala | 82 ++
.../params/SqoopDataSourceParamsResolver.scala | 143 ++++
.../engineplugin/trino/TrinoEngineConnPlugin.java | 78 --
.../TrinoProcessEngineConnLaunchBuilder.java | 35 -
.../trino/conf/TrinoConfiguration.java | 90 --
.../engineplugin/trino/conf/TrinoEngineConfig.java | 53 --
.../trino/exception/TrinoClientException.java | 26 -
.../trino/exception/TrinoGrantmaException.java | 26 -
.../exception/TrinoModifySchemaException.java | 26 -
.../exception/TrinoStateInvalidException.java | 26 -
.../trino/executor/TrinoEngineConnExecutor.java | 561 ------------
.../trino/password/CommandPasswordCallback.java | 75 ++
.../trino/password/StaticPasswordCallback.java | 40 +
.../linkis/engineplugin/trino/utils/TrinoCode.java | 47 -
.../engineplugin/trino/utils/TrinoSQLHook.java | 34 -
.../main/resources/linkis-engineconn.properties | 1 -
.../trino/src/main/resources/log4j2.xml | 3 +
.../engineplugin/trino/TrinoEngineConnPlugin.scala | 66 ++
.../TrinoProcessEngineConnLaunchBuilder.scala | 37 +
.../trino/conf/TrinoConfiguration.scala | 73 ++
.../trino/conf/TrinoEngineConfig.scala | 48 ++
.../trino/exception/TrinoException.scala | 32 +
.../trino/executor/TrinoEngineConnExecutor.scala | 563 ++++++++++++
.../engineplugin/trino/utils/TrinoCode.scala | 65 ++
.../engineplugin/trino/utils/TrinoSQLHook.scala | 36 +
linkis-extensions/linkis-et-monitor/pom.xml | 23 +-
.../src/main/assembly/distribution.xml | 3 -
.../cleaner/service/impl/CleanerServiceImpl.java | 3 +-
.../monitor/config/ApplicationConfiguration.java | 48 ++
.../linkis/monitor/config/MonitorConfig.java | 5 +-
.../monitor/jobhistory/entity/JobHistory.java | 186 +++-
.../apache/linkis/monitor/scheduled/BmlClear.java | 17 +
.../monitor/scheduled/EntranceTaskMonitor.java | 47 +-
.../linkis/monitor/scheduled/JobHistoryClear.java | 1 +
.../monitor/scheduled/JobHistoryMonitor.java | 127 +--
.../linkis/monitor/scheduled/ResourceClear.java | 49 ++
.../linkis/monitor/scheduled/ResourceMonitor.java | 199 +++--
.../linkis/monitor/scheduled/TaskArchiveClear.java | 53 ++
.../linkis/monitor/until/JobMonitorUtils.java | 88 ++
.../mapper/common/InsLabelRelationMapper.xml | 15 +-
.../resources/mapper/common/InstanceInfoMapper.xml | 13 +-
.../mapper/common/InstanceLabelMapper.xml | 13 +-
.../resources/mapper/common/JobHistoryMapper.xml | 74 +-
.../LinkisJobHistoryScanSpringConfiguration.scala | 5 +-
.../linkis/monitor/constants/Constants.scala | 12 +-
.../monitor/jobhistory/JobHistoryDataFetcher.scala | 5 +-
.../errorcode/JobHistoryErrCodeRule.scala | 25 +-
.../jobhistory/index/JobIndexHitEvent.scala | 22 +
.../monitor/jobhistory/index/JobIndexRule.scala | 82 ++
.../monitor/jobhistory/index/JobIndexSender.scala | 25 +
.../jobtime/JobTimeExceedAlertSender.scala | 3 +-
.../labels/JobHistoryLabelsAlertSender.scala | 6 +-
.../jobhistory/labels/JobHistoryLabelsRule.scala | 76 +-
.../jobhistory/runtime/CommonJobRunTimeRule.scala | 15 +-
.../jobhistory/runtime/JobHistoryRunTimeRule.scala | 15 +-
.../monitor/utils/alert/ims/ImsAlertDesc.scala | 32 +-
.../utils/alert/ims/ImsAlertPropFileData.scala | 1 +
.../monitor/utils/alert/ims/ImsRequest.scala | 1 +
.../utils/alert/ims/MonitorAlertUtils.scala | 54 +-
.../utils/alert/ims/UserLabelAlertUtils.scala | 47 -
.../utils/alert/PooledImsAlertSenderTest.java | 3 +
.../src/test/resources/log4j2-console.xml | 13 +-
.../linkis/storage/io/client/DefaultIOClient.scala | 4 +-
.../io/iteraceptor/IOMethodInterceptor.scala | 14 +-
.../IOMethodInterceptorCreatorImpl.scala | 4 +-
.../linkis/storage/io/utils/IOClientUtils.scala | 3 +-
.../DefaultCodeExecTaskExecutorManager.scala | 7 +-
.../computation/monitor/EngineConnMonitor.scala | 3 +-
.../physical/CodeLogicalUnitExecTask.scala | 6 +-
.../service/ComputationTaskExecutionReceiver.scala | 10 +-
.../ecm/ComputationEngineConnManager.scala | 54 +-
.../ecm/cache/EngineAsyncResponseCache.scala | 16 +-
.../orchestrator/ecm/conf/ECMPluginConf.scala | 2 +
.../impl/ComputationEngineConnExecutor.scala | 2 +-
.../impl/DefaultEngineAsyncResponseService.scala | 14 +-
.../orchestrator/execution/AbstractExecution.scala | 26 -
.../execution/impl/DefaultTaskManager.scala | 47 +-
.../execution/impl/ExecutionImpl.scala | 27 +-
.../listener/execution/ExecutionTaskEvent.scala | 13 +-
.../execution/ExecutionTaskStatusListener.scala | 28 -
.../linkis/bml/service/impl/TaskServiceImpl.java | 11 +-
.../resources/mapper/common/BmlProjectMapper.xml | 62 ++
.../resources/mapper/common/DownloadMapper.xml | 6 +-
.../resources/mapper/common/ResourceMapper.xml | 107 +++
.../main/resources/mapper/common/TaskMapper.xml | 69 ++
.../main/resources/mapper/common/VersionMapper.xml | 194 +++++
.../resources/mapper/mysql/BmlProjectMapper.xml | 78 --
.../main/resources/mapper/mysql/ResourceMapper.xml | 121 ---
.../src/main/resources/mapper/mysql/TaskMapper.xml | 69 --
.../main/resources/mapper/mysql/VersionMapper.xml | 243 ------
.../mapper/postgresql/BmlProjectMapper.xml | 81 --
.../resources/mapper/postgresql/ResourceMapper.xml | 110 ---
.../resources/mapper/postgresql/TaskMapper.xml | 69 --
.../resources/mapper/postgresql/VersionMapper.xml | 187 ----
.../apache/linkis/bml/dao/BmlProjectDaoTest.java | 5 -
.../src/test/resources/application.properties | 5 +-
.../src/test/resources/create_pg.sql | 129 ---
.../configuration/dao/AcrossClusterRuleMapper.java | 3 +-
.../restful/api/AcrossClusterRuleRestfulApi.java | 5 +-
.../restful/api/ConfigurationRestfulApi.java | 14 +-
.../restful/api/TemplateRestfulApi.java | 3 +
.../restful/api/TenantConfigrationRestfulApi.java | 9 +-
.../restful/api/UserIpConfigrationRestfulApi.java | 2 +-
.../service/AcrossClusterRuleService.java | 7 +-
.../configuration/service/TenantConfigService.java | 2 +
.../service/impl/AcrossClusterRuleServiceImpl.java | 10 +-
.../service/impl/TemplateConfigKeyServiceImpl.java | 5 +-
.../service/impl/TenantConfigServiceImpl.java | 8 +-
.../service/impl/UserIpConfigServiceImpl.java | 2 +-
.../mapper/common/AcrossClusterRuleMapper.xml | 205 +++++
.../ConfigKeyLimitForUserMapper.xml | 0
.../main/resources/mapper/common/ConfigMapper.xml | 485 +++++++++++
.../mapper/common/DepartmentTenantMapper.xml | 96 +++
.../main/resources/mapper/common/LabelMapper.xml | 18 +-
.../{mysql => common}/TemplateConfigKeyMapper.xml | 0
.../main/resources/mapper/common/UserIpMapper.xml | 91 ++
.../resources/mapper/common/UserTenantMapper.xml | 95 +++
.../mapper/mysql/AcrossClusterRuleMapper.xml | 203 -----
.../main/resources/mapper/mysql/ConfigMapper.xml | 485 -----------
.../main/resources/mapper/mysql/UserIpMapper.xml | 91 --
.../resources/mapper/mysql/UserTenantMapper.xml | 95 ---
.../mapper/postgresql/AcrossClusterRuleMapper.xml | 130 ---
.../postgresql/ConfigKeyLimitForUserMapper.xml | 145 ----
.../resources/mapper/postgresql/ConfigMapper.xml | 485 -----------
.../mapper/postgresql/TemplateConfigKeyMapper.xml | 161 ----
.../resources/mapper/postgresql/UserIpMapper.xml | 91 --
.../mapper/postgresql/UserTenantMapper.xml | 92 --
.../linkis/configuration/conf/Configuration.scala | 8 -
.../linkis/configuration/constant/Constants.scala | 2 +-
.../configuration/service/CategoryService.scala | 7 +-
.../linkis/configuration/dao/ConfigMapperTest.java | 39 +-
.../linkis/configuration/dao/LabelMapperTest.java | 17 +
.../exception/ConfigurationExceptionTest.java | 17 +
.../restful/api/ConfigurationRestfulApiTest.java | 24 +-
.../src/test/resources/application.properties | 11 +-
.../src/test/resources/create.sql | 2 +-
.../src/test/resources/create_pg.sql | 148 ----
.../ContextValueTypeConditionParser.java | 19 +-
.../org/apache/linkis/cs/conf/CSConfiguration.java | 28 +
.../mapper/common/contextHistoryMapper.xml | 79 ++
.../mapper/common/contextIDListenerMapper.xml | 46 +
.../resources/mapper/common/contextIDMapper.xml | 105 +++
.../mapper/common/contextKeyListenerMapper.xml | 4 +-
.../resources/mapper/common/contextMapMapper.xml | 168 ++++
.../mapper/mysql/contextHistoryMapper.xml | 89 --
.../mapper/mysql/contextIDListenerMapper.xml | 51 --
.../resources/mapper/mysql/contextIDMapper.xml | 105 ---
.../resources/mapper/mysql/contextMapMapper.xml | 168 ----
.../mapper/postgresql/contextHistoryMapper.xml | 83 --
.../mapper/postgresql/contextIDListenerMapper.xml | 50 --
.../mapper/postgresql/contextIDMapper.xml | 110 ---
.../mapper/postgresql/contextMapMapper.xml | 172 ----
.../persistence/dao/ContextHistoryMapperTest.java | 6 +-
.../cs/persistence/dao/ContextIDMapperTest.java | 12 +-
.../cs/persistence/dao/ContextMapMapperTest.java | 266 ------
.../cs/server/conf/ContextServerConfTest.java | 1 +
.../src/test/resources/application.properties | 5 +-
.../src/test/resources/application.yml | 13 -
.../src/test/resources/create_pg.sql | 90 --
.../instance/label/InsLabelAutoConfiguration.java | 6 +-
.../instance/label/restful/InstanceRestful.java | 23 +-
.../label/service/impl/SpringInsLabelService.java | 2 -
.../mapper/common/InsLabelRelationMapper.xml | 205 +++++
.../resources/mapper/common/InstanceInfoMapper.xml | 53 ++
.../mapper/common/InstanceLabelMapper.xml | 115 +++
.../mapper/mysql/InsLabelRelationMapper.xml | 206 -----
.../resources/mapper/mysql/InstanceInfoMapper.xml | 55 --
.../resources/mapper/mysql/InstanceLabelMapper.xml | 115 ---
.../mapper/postgresql/InsLabelRelationMapper.xml | 205 -----
.../mapper/postgresql/InstanceInfoMapper.xml | 55 --
.../mapper/postgresql/InstanceLabelMapper.xml | 114 ---
.../label/dao/InsLabelRelationDaoTest.java | 10 -
.../instance/label/dao/InstanceInfoDaoTest.java | 91 --
.../instance/label/dao/InstanceLabelDaoTest.java | 5 -
.../src/test/resources/application.properties | 3 +-
.../src/test/resources/create_pg.sql | 65 --
.../linkis/jobhistory/dao/JobHistoryMapper.java | 29 +-
.../linkis/jobhistory/entity/JobHistory.java | 159 ++--
.../linkis/jobhistory/entity/QueryTaskVO.java | 46 +
.../jobhistory/restful/api/QueryRestfulApi.java | 427 +++++++--
.../linkis/jobhistory/util/JobhistoryUtils.java | 169 ++++
.../resources/mapper/common/JobDetailMapper.xml | 22 +-
.../resources/mapper/mysql/JobHistoryMapper.xml | 121 ++-
.../jobhistory/conf/JobhistoryConfiguration.scala | 5 +
.../jobhistory/conversions/TaskConversions.scala | 63 +-
.../jobhistory/service/JobHistoryQueryService.java | 16 +-
.../service/impl/JobHistoryQueryServiceImpl.scala | 78 +-
.../apache/linkis/jobhistory/util/QueryUtils.scala | 68 +-
.../linkis/jobhistory/dao/JobDetailMapperTest.java | 95 ---
.../jobhistory/dao/JobHistoryMapperTest.java | 8 +-
.../restful/api/QueryRestfulApiTest.java | 179 ----
.../service/JobHistoryQueryServiceTest.java | 12 +-
.../src/test/resources/application.properties | 8 +-
.../src/test/resources/create.sql | 4 +-
.../src/test/resources/create_pg.sql | 62 --
.../errorcode/client/ClientConfiguration.java | 2 +-
.../apache/linkis/bml/conf/BmlConfiguration.scala | 2 +-
.../linkis/cs/client/utils/ContextClientConf.scala | 2 +-
.../client/config/DatasourceClientConfig.scala | 2 +-
.../filesystem/conf/WorkspaceClientConf.scala | 2 +-
.../errorcode/BmlClientErrorCodeSummaryTest.java | 45 -
.../client/builder/ContextClientFactoryTest.java | 35 -
.../builder/HttpContextClientConfigTest.java | 33 -
.../errorcode/CsClientErrorCodeSummaryTest.java | 59 --
.../org/apache/linkis/cs/client/test/Test.java | 134 ---
.../cs/client/test/bean/ClientTestContextKey.java | 67 --
.../client/test/bean/ClientTestContextValue.java | 47 -
.../cs/client/test/bean/ResourceValueBean.java | 45 -
.../test/listener/CommonContextKeyListener.java | 40 -
.../client/test/no_context_search/TestClear.java | 60 --
.../cs/client/test/restful/ListenerTest.java | 20 -
.../linkis/cs/client/test/restful/RestfulTest.java | 146 ----
.../linkis/cs/client/test/service/TestInfo.java | 99 ---
.../linkis/cs/client/test/service/TestRemove.java | 113 ---
.../cs/client/test/service/TestSearchService.java | 113 ---
.../test/test_multiuser/TestChangeContext.java | 110 ---
.../test/test_multiuser/TestCreateContext.java | 122 ---
.../cs/client/utils/ContextClientConfTest.java | 36 -
.../cs/client/utils/ContextServiceUtilsTest.java | 103 ---
.../cs/client/utils/SerializeHelperTest.java | 60 --
.../datasource/client/TestDataSourceClient.scala | 123 ---
.../linkis/datasource/client/TestHiveClient.scala | 144 ----
.../linkis/datasource/client/TestMysqlClient.scala | 149 ----
.../client/config/DatasourceClientConfigTest.java | 49 --
.../DataSourceClientBuilderExceptionTest.java | 34 -
.../imp/LinkisDataSourceRemoteClientTest.java | 37 -
.../errorcode/client/ClientConfigurationTest.java | 46 -
.../client/ErrorCodeClientBuilderTest.java | 42 -
.../client/LinkisErrorCodeClientTest.java | 44 -
.../client/handler/ErrorCodeHandlerTest.java | 37 -
.../client/handler/LinkisErrorCodeHandlerTest.java | 48 --
.../client/manager/LinkisErrorCodeManagerTest.java | 45 -
.../LinkisErrorCodeSynchronizerTest.java | 46 -
.../client/action/ErrorCodeActionTest.scala | 41 -
.../filesystem/WorkspaceClientFactoryTest.scala | 36 -
.../filesystem/WorkspaceClientImplTest.scala | 36 -
.../action/OpenScriptFromBMLActionTest.scala | 35 -
.../filesystem/conf/WorkspaceClientConfTest.scala | 45 -
.../response/ScriptFromBMLResponseTest.scala | 38 -
.../linkis/udf/entity/PythonModuleInfoVO.java | 209 +++++
.../udf/api/rpc/RequestPythonModuleProtocol.scala | 28 +
.../udf/api/rpc/ResponsePythonModuleProtocol.scala | 33 +
.../entity/enumeration/ContextScopeTest.java | 40 -
.../common/entity/enumeration/ContextTypeTest.java | 48 --
.../cs/common/entity/enumeration/DBTypeTest.java | 40 -
.../common/entity/enumeration/ExpireTypeTest.java | 38 -
.../cs/common/entity/enumeration/WorkTypeTest.java | 40 -
.../cs/common/protocol/ContextHistoryTypeTest.java | 36 -
.../cs/common/protocol/ContextIDTypeTest.java | 43 -
.../cs/common/protocol/ContextKeyTypeTest.java | 37 -
.../common/protocol/ContextKeyValueTypeTest.java | 37 -
.../cs/common/protocol/ContextValueTypeTest.java | 37 -
.../helper/ContextSerializationHelperTest.java | 199 -----
.../CombinedNodeIDContextIDSerializerTest.java | 59 --
.../context/CommonContextKeySerializerTest.java | 60 --
.../value/LinkisBMLResourceSerializerTest.java | 63 --
.../value/data/CSResultDataSerializerTest.java | 60 --
.../value/data/LinkisJobDataSerializerTest.java | 61 --
.../impl/value/metadata/CSTableSerializerTest.java | 65 --
.../value/object/CSFlowInfosSerializerTest.java | 57 --
.../test/ContextSerializationHelperTest.java | 187 ----
.../linkis/cs/common/utils/CSCommonUtilsTest.java | 67 --
.../cs/common/utils/CSHighAvailableUtilsTest.java | 64 --
.../linkis/cs/listener/test/TestContextID.java | 35 -
.../linkis/cs/listener/test/TestContextKey.java | 71 --
.../cs/listener/test/TestContextKeyValue.java | 49 --
.../linkis/cs/listener/test/TestContextValue.java | 44 -
.../linkis/errorcode/common/CommonConfTest.java | 33 -
.../query/common/MdmConfigurationTest.java | 37 -
.../query/common/cache/CacheConfigurationTest.java | 40 -
.../query/common/cache/ConnCacheManagerTest.java | 48 --
.../exception/MetaMethodInvokeExceptionTest.java | 34 -
.../common/exception/MetaRuntimeExceptionTest.java | 34 -
.../linkis/udf/excepiton/UDFExceptionTest.java | 33 -
.../apache/linkis/udf/utils/ConstantVarTest.java | 48 --
.../errorcode/common/LinkisErrorCodeTest.scala | 35 -
.../server/response/EngineLabelResponse.java | 2 +-
.../server/restful/GatewayAuthTokenRestfulApi.java | 46 +
.../server/service/GatewayAuthTokenService.java | 2 +
.../service/impl/GatewayAuthTokenServiceImpl.java | 8 +
.../filesystem/conf/WorkSpaceConfiguration.java | 15 +-
.../filesystem/constant/WorkSpaceConstants.java | 8 +
.../exception/WorkspaceExceptionManager.java | 18 +
.../filesystem/restful/api/BMLFsRestfulApi.java | 11 +-
.../filesystem/restful/api/FsRestfulApi.java | 557 ++++++++++--
.../linkis/filesystem/util/WorkspaceUtil.java | 4 +
.../mapper/common/CgManagerLabelMapper.xml | 6 +-
.../mapper/common/ConfigurationConfigKeyMapper.xml | 41 +
.../common/ConfigurationConfigValueMapper.xml | 11 +-
.../ConfigurationKeyEngineRelationMapper.xml | 5 +-
.../mapper/common/DatasourceAccessMapper.xml | 49 ++
.../mapper/common/DatasourceEnvMapper.xml | 51 ++
.../mapper/common/DatasourceTypeKeyMapper.xml | 64 ++
.../mapper/common/DatasourceTypeMapper.xml | 50 ++
.../common/EngineConnPluginBmlResourcesMapper.xml | 6 +-
.../mapper/common/GatewayAuthTokenMapper.xml | 51 ++
.../resources/mapper/common/PsErrorCodeMapper.xml | 47 +
.../common/RmExternalResourceProviderMapper.xml | 47 +
.../resources/mapper/common/UdfBaseInfoMapper.xml | 24 +
.../resources/mapper/common/UdfManagerMapper.xml | 40 +
.../main/resources/mapper/common/UdfTreeMapper.xml | 51 ++
.../src/main/resources/mapper/common/VarMapper.xml | 60 ++
.../mapper/mysql/ConfigurationConfigKeyMapper.xml | 31 -
.../mapper/mysql/DatasourceAccessMapper.xml | 49 --
.../resources/mapper/mysql/DatasourceEnvMapper.xml | 51 --
.../mapper/mysql/DatasourceTypeKeyMapper.xml | 64 --
.../mapper/mysql/DatasourceTypeMapper.xml | 50 --
.../mapper/mysql/GatewayAuthTokenMapper.xml | 51 --
.../resources/mapper/mysql/PsErrorCodeMapper.xml | 47 -
.../mysql/RmExternalResourceProviderMapper.xml | 47 -
.../resources/mapper/mysql/UdfManagerMapper.xml | 40 -
.../main/resources/mapper/mysql/UdfTreeMapper.xml | 51 --
.../src/main/resources/mapper/mysql/VarMapper.xml | 64 --
.../postgresql/ConfigurationConfigKeyMapper.xml | 68 --
.../mapper/postgresql/DatasourceAccessMapper.xml | 49 --
.../mapper/postgresql/DatasourceEnvMapper.xml | 51 --
.../mapper/postgresql/DatasourceTypeKeyMapper.xml | 64 --
.../mapper/postgresql/DatasourceTypeMapper.xml | 50 --
.../mapper/postgresql/GatewayAuthTokenMapper.xml | 51 --
.../mapper/postgresql/PsErrorCodeMapper.xml | 47 -
.../RmExternalResourceProviderMapper.xml | 47 -
.../mapper/postgresql/UdfManagerMapper.xml | 40 -
.../resources/mapper/postgresql/UdfTreeMapper.xml | 51 --
.../main/resources/mapper/postgresql/VarMapper.xml | 61 --
.../linkis/filesystem/service/FsService.scala | 20 +
.../filesystem/validator/PathValidator.scala | 16 +-
.../apache/linkis/basedatamanager/server/Scan.java | 26 -
.../server/WebApplicationServer.java | 34 -
.../basedatamanager/server/dao/BaseDaoTest.java | 31 -
.../server/dao/CgManagerLabelMapperTest.java | 45 -
.../dao/ConfigurationConfigKeyMapperTest.java | 86 --
.../dao/ConfigurationConfigValueMapperTest.java | 65 --
.../ConfigurationKeyEngineRelationMapperTest.java | 53 --
.../server/dao/DatasourceAccessMapperTest.java | 48 --
.../server/dao/DatasourceEnvMapperTest.java | 55 --
.../server/dao/DatasourceTypeKeyMapperTest.java | 39 -
.../server/dao/DatasourceTypeMapperTest.java | 39 -
.../EngineConnPluginBmlResourcesMapperTest.java | 56 --
.../server/dao/GatewayAuthTokenMapperTest.java | 55 --
.../server/dao/PsErrorCodeMapperTest.java | 50 --
.../dao/RmExternalResourceProviderMapperTest.java | 52 --
.../server/dao/UdfManagerMapperTest.java | 47 -
.../server/dao/UdfTreeMapperTest.java | 54 --
.../ConfigurationTemplateRestfulApiTest.java | 143 ----
.../restful/DatasourceAccessRestfulApiTest.java | 149 ----
.../restful/DatasourceEnvRestfulApiTest.java | 138 ---
.../restful/DatasourceTypeKeyRestfulApiTest.java | 158 ----
.../restful/DatasourceTypeRestfulApiTest.java | 159 ----
.../server/restful/ErrorCodeRestfulApiTest.java | 154 ----
.../restful/GatewayAuthTokenRestfulApiTest.java | 161 ----
.../basedatamanager/server/restful/MvcUtils.java | 126 ---
.../RmExternalResourceProviderRestfulApiTest.java | 154 ----
.../server/restful/UdfManagerRestfulApiTest.java | 151 ----
.../server/restful/UdfTreeRestfulApiTest.java | 159 ----
.../service/DatasourceAccessServiceTest.java | 63 --
.../server/service/DatasourceEnvServiceTest.java | 65 --
.../service/DatasourceTypeKeyServiceTest.java | 65 --
.../server/service/DatasourceTypeServiceTest.java | 64 --
.../server/service/ErrorCodeServiceTest.java | 63 --
.../service/GatewayAuthTokenServiceTest.java | 63 --
.../RmExternalResourceProviderServiceTest.java | 60 --
.../server/service/UdfManagerServiceTest.java | 59 --
.../server/service/UdfTreeServiceTest.java | 60 --
.../org/apache/linkis/errorcode/server/Scan.java | 26 -
.../errorcode/server/WebApplicationServer.java | 34 -
.../linkis/errorcode/server/dao/BaseDaoTest.java | 31 -
.../errorcode/server/dao/ErrorCodeMapperTest.java | 40 -
.../filesystem/restful/api/FsRestfulApiTest.java | 366 ++++----
.../test/java/org/apache/linkis/variable/Scan.java | 26 -
.../linkis/variable/WebApplicationServer.java | 34 -
.../apache/linkis/variable/dao/BaseDaoTest.java | 31 -
.../apache/linkis/variable/dao/VarMapperTest.java | 124 ---
.../apache/linkis/variable/restful/MvcUtils.java | 115 ---
.../restful/api/VariableRestfulApiTest.java | 112 ---
.../variable/service/VariableServiceTest.java | 85 --
.../src/test/resources/application.properties | 29 +-
.../src/test/resources/basedata_manager_create.sql | 451 ----------
.../test/resources/basedata_manager_create_pg.sql | 252 ------
.../src/test/resources/error_code_create.sql | 34 -
.../src/test/resources/error_code_create_pg.sql | 32 -
.../src/test/resources/info.text | 19 -
.../src/test/resources/variable_create.sql | 61 --
.../src/test/resources/variable_create_pg.sql | 59 --
.../common/lock/dao/CommonLockMapperTest.java | 13 +
.../src/test/resources/application.properties | 10 +-
.../src/test/resources/create.sql | 6 +-
.../src/test/resources/create_pg.sql | 31 -
.../src/test/resources/data.sql | 22 +
.../org/apache/linkis/udf/api/UDFRestfulApi.java | 280 ++++++
.../linkis/udf/dao/PythonModuleInfoMapper.java | 45 +
.../java/org/apache/linkis/udf/dao/UDFDao.java | 4 +-
.../apache/linkis/udf/entity/PythonModuleInfo.java | 158 ++++
.../udf/service/PythonModuleInfoService.java | 41 +
.../org/apache/linkis/udf/service/UDFService.java | 2 +-
.../service/impl/PythonModuleInfoServiceImpl.java | 64 ++
.../linkis/udf/service/impl/UDFServiceImpl.java | 4 +-
.../mapper/common/PythonModuleInfoMapper.xml | 93 ++
.../src/main/resources/mapper/common/UDFDao.xml | 426 +++++++++
.../main/resources/mapper/common/UDFTreeDao.xml | 56 +-
.../main/resources/mapper/common/UDFVersionDao.xml | 115 +++
.../src/main/resources/mapper/mysql/UDFDao.xml | 471 ----------
.../main/resources/mapper/mysql/UDFVersionDao.xml | 130 ---
.../main/resources/mapper/postgresql/UDFDao.xml | 440 ----------
.../resources/mapper/postgresql/UDFVersionDao.xml | 119 ---
.../apache/linkis/udf/api/rpc/UdfReceiver.scala | 56 +-
.../linkis/udf/api/rpc/UdfReceiverChooser.scala | 9 +-
.../linkis/udf/dao/PythonModuleInfoMapperTest.java | 112 +++
.../udf/service/PythonModuleInfoServiceTest.java | 129 +++
.../src/test/resources/application.properties | 11 +-
.../src/test/resources/create.sql | 71 +-
.../src/test/resources/create_pg.sql | 143 ----
.../linkis-udf-service/src/test/resources/data.sql | 61 ++
.../authentication/conf/TokenConfiguration.scala | 4 +-
.../service/CachedTokenService.scala | 2 +-
.../gateway/authentication/dao/TokenDaoTest.java | 2 +-
.../service/CachedTokenServiceTest.java | 2 +-
.../src/test/resources/create.sql | 2 +-
.../src/test/resources/create_pg.sql | 2 +-
.../gateway/config/GatewayConfiguration.scala | 7 +-
.../linkis/gateway/security/UserRestful.scala | 12 +
.../httpclient/dws/config/DWSClientConfig.scala | 9 +-
.../ujes/parser/ECMRequestGatewayParser.scala | 74 ++
.../ujes/parser/EntranceRequestGatewayParser.scala | 14 +-
.../ujes/route/AbstractLabelGatewayRouter.scala | 31 +-
.../springcloud/http/IpPriorityLoadBalancer.java | 19 +-
linkis-web/src/apps/URM/module/header/index.vue | 12 +-
linkis-web/src/apps/linkis/module/header/index.vue | 8 +-
linkis-web/src/dss/module/header/index.vue | 8 +-
tool/dependencies/known-dependencies.txt | 12 +-
1259 files changed, 36182 insertions(+), 50957 deletions(-)
diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala
index ec65fd0a67..822bc2aa07 100644
--- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala
+++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala
@@ -81,7 +81,7 @@ object Configuration extends Logging {
"The request interface %s is abnormal. You can try to troubleshoot common problems in the knowledge base document"
)
- val LINKIS_TOKEN = CommonVars("wds.linkis.token", "LINKIS-AUTH-eTaYLbQpmIulPyrXcMl")
+ val LINKIS_TOKEN = CommonVars("wds.linkis.token", "LINKIS-AUTH")
val GLOBAL_CONF_CHN_NAME = "全局设置"
diff --git a/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala b/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala
index 3ebbbc33ba..e18837fd5f 100644
--- a/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala
+++ b/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala
@@ -152,11 +152,11 @@ object HDFSUtils extends Logging {
hdfsFileSystemContainer.updateLastAccessTime
hdfsFileSystemContainer.getFileSystem
} else {
- getHDFSUserFileSystem(userName, label, getConfiguration(userName, label))
+ getHDFSUserFileSystem(userName, label, getConfigurationByLabel(userName, label))
}
}
} else {
- getHDFSUserFileSystem(userName, label, getConfiguration(userName, label))
+ getHDFSUserFileSystem(userName, label, getConfigurationByLabel(userName, label))
}
}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/FSFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/FSFactory.java
deleted file mode 100644
index 34a92ead85..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/FSFactory.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage;
-
-import org.apache.linkis.common.io.Fs;
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.storage.exception.StorageWarnException;
-import org.apache.linkis.storage.factory.BuildFactory;
-import org.apache.linkis.storage.utils.StorageConfiguration;
-import org.apache.linkis.storage.utils.StorageUtils;
-
-import java.text.MessageFormat;
-import java.util.Map;
-
-import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.UNSUPPORTED_FILE;
-
-public class FSFactory {
- private static final Map<String, BuildFactory> buildClasses =
- StorageUtils.loadClass(
- StorageConfiguration.STORAGE_BUILD_FS_CLASSES.getValue(), t -> t.fsName());
-
- public static BuildFactory getBuildFactory(String fsName) {
- if (!buildClasses.containsKey(fsName)) {
- throw new StorageWarnException(
- UNSUPPORTED_FILE.getErrorCode(),
- MessageFormat.format(UNSUPPORTED_FILE.getErrorDesc(), fsName));
- }
- return buildClasses.get(fsName);
- }
-
- public static Fs getFs(String fsType, String proxyUser) {
- String user = StorageUtils.getJvmUser();
- return getBuildFactory(fsType).getFs(user, proxyUser);
- }
-
- public static Fs getFs(String fsType) {
- String user = StorageUtils.getJvmUser();
- return getBuildFactory(fsType).getFs(user, user);
- }
-
- /**
- * 1. If this machine has shared storage, the file:// type FS obtained here is the FS of the
- * process user. 2. If this machine does not have shared storage, then the file:// type FS
- * obtained is the proxy to the Remote (shared storage machine root) FS. 3. If it is HDFS, it
- * returns the FS of the process user. 1、如果这台机器装有共享存储则这里获得的file://类型的FS为该进程用户的FS
- * 2、如果这台机器没有共享存储则获得的file://类型的FS为代理到Remote(共享存储机器root)的FS 3、如果是HDFS则返回的就是该进程用户的FS
- *
- * @param fsPath
- * @return
- */
- public static Fs getFs(FsPath fsPath) {
- return getFs(fsPath.getFsType());
- }
-
- /**
- * 1. If the process user is passed and the proxy user and the process user are consistent, the
- * file:// type FS is the FS of the process user (the shared storage exists) 2. If the process
- * user is passed and the proxy user and the process user are consistent and there is no shared
- * storage, the file:// type FS is the proxy to the remote (shared storage machine root) FS 3. If
- * the passed proxy user and process user are consistent, the hdfs type is the FS of the process
- * user. 4. If the proxy user and the process user are inconsistent, the hdfs type is the FS after
- * the proxy.
- *
- * @param fsPath
- * @param proxyUser
- * @return
- */
- public static Fs getFsByProxyUser(FsPath fsPath, String proxyUser) {
- return getFs(fsPath.getFsType(), proxyUser);
- }
-
- public Fs getFSByLabel(String fs, String label) {
- String user = StorageUtils.getJvmUser();
- return getBuildFactory(fs).getFs(user, user, label);
- }
-
- public Fs getFSByLabelAndUser(String fs, String label, String proxy) {
- String user = StorageUtils.getJvmUser();
- return getBuildFactory(fs).getFs(user, proxy, label);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/LineMetaData.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/LineMetaData.java
deleted file mode 100644
index a6dab8b37d..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/LineMetaData.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage;
-
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.storage.resultset.ResultMetaData;
-
-public class LineMetaData implements ResultMetaData {
-
- private String metaData = null;
-
- public LineMetaData() {}
-
- public LineMetaData(String metaData) {
- this.metaData = metaData;
- }
-
- public String getMetaData() {
- return metaData;
- }
-
- public void setMetaData(String metaData) {
- this.metaData = metaData;
- }
-
- public MetaData cloneMeta() {
- return new LineMetaData(metaData);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/LineRecord.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/LineRecord.java
deleted file mode 100644
index 7f62090005..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/LineRecord.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage;
-
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.storage.resultset.ResultRecord;
-
-public class LineRecord implements ResultRecord {
- private String line;
-
- public LineRecord(String line) {
- this.line = line;
- }
-
- public String getLine() {
- return line;
- }
-
- public void setLine(String line) {
- this.line = line;
- }
-
- @Override
- public Record cloneRecord() {
- return new LineRecord(line);
- }
-
- @Override
- public String toString() {
- return line;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java
deleted file mode 100644
index 7fb5c1b4ac..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.conf;
-
-import org.apache.linkis.common.conf.CommonVars;
-import org.apache.linkis.common.utils.ByteTimeUtils;
-
-import org.apache.commons.lang3.StringUtils;
-
-public class LinkisStorageConf {
- private static final Object CONF_LOCK = new Object();
-
- public static final String DOLPHIN = "dolphin";
-
- public static final String PARQUET = "parquet";
-
- public static final String PARQUET_FILE_SUFFIX = ".parquet";
-
- public static final String ORC = "orc";
-
- public static final String ORC_FILE_SUFFIX = ".orc";
-
- public static final String HDFS_FILE_SYSTEM_REST_ERRS =
- CommonVars.apply(
- "wds.linkis.hdfs.rest.errs",
- ".*Filesystem closed.*|.*Failed to find any Kerberos tgt.*")
- .getValue();
-
- public static final String ROW_BYTE_MAX_LEN_STR =
- CommonVars.apply("wds.linkis.resultset.row.max.str", "2m").getValue();
-
- public static final String ENGINE_RESULT_TYPE =
- CommonVars.apply("linkis.engine.resultSet.type", DOLPHIN, "Result type").getValue();
-
- public static final long ROW_BYTE_MAX_LEN = ByteTimeUtils.byteStringAsBytes(ROW_BYTE_MAX_LEN_STR);
-
- public static final String FILE_TYPE =
- CommonVars.apply(
- "wds.linkis.storage.file.type",
- "dolphin,sql,scala,py,hql,python,out,log,text,txt,sh,jdbc,ngql,psql,fql,tsql"
- + ","
- + PARQUET
- + ","
- + ORC)
- .getValue();
-
- private static volatile String[] fileTypeArr = null;
-
- private static String[] fileTypeArrParser(String fileType) {
- if (StringUtils.isBlank(fileType)) {
- return new String[0];
- } else {
- return fileType.split(",");
- }
- }
-
- public static String[] getFileTypeArr() {
- if (fileTypeArr == null) {
- synchronized (CONF_LOCK) {
- if (fileTypeArr == null) {
- fileTypeArr = fileTypeArrParser(FILE_TYPE);
- }
- }
- }
- return fileTypeArr;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/CSVFsWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/CSVFsWriter.java
deleted file mode 100644
index f05e93c657..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/CSVFsWriter.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.csv;
-
-import org.apache.linkis.common.io.FsWriter;
-
-import java.io.OutputStream;
-
-public abstract class CSVFsWriter extends FsWriter {
- public abstract String getCharset();
-
- public abstract String getSeparator();
-
- public abstract boolean isQuoteRetouchEnable();
-
- public static CSVFsWriter getCSVFSWriter(
- String charset, String separator, boolean quoteRetouchEnable, OutputStream outputStream) {
- return new StorageCSVWriter(charset, separator, quoteRetouchEnable, outputStream);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/StorageCSVWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/StorageCSVWriter.java
deleted file mode 100644
index d98be40337..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/StorageCSVWriter.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.csv;
-
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.storage.domain.Column;
-import org.apache.linkis.storage.domain.DataType;
-import org.apache.linkis.storage.resultset.table.TableMetaData;
-import org.apache.linkis.storage.resultset.table.TableRecord;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.stream.Stream;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class StorageCSVWriter extends CSVFsWriter {
- private static final Logger logger = LoggerFactory.getLogger(StorageCSVWriter.class);
-
- private final String charset;
- private final String separator;
- private final boolean quoteRetouchEnable;
- private final OutputStream outputStream;
-
- private final String delimiter;
- private final StringBuilder buffer;
-
- public StorageCSVWriter(
- String charset, String separator, boolean quoteRetouchEnable, OutputStream outputStream) {
- this.charset = charset;
- this.separator = separator;
- this.quoteRetouchEnable = quoteRetouchEnable;
- this.outputStream = outputStream;
-
- if (StringUtils.isBlank(separator)) {
- this.delimiter = "\t";
- } else {
- switch (separator) {
- case "t":
- this.delimiter = "\t";
- break;
- default:
- this.delimiter = separator;
- break;
- }
- }
- this.buffer = new StringBuilder(50000);
- }
-
- @Override
- public String getCharset() {
- return charset;
- }
-
- @Override
- public String getSeparator() {
- return separator;
- }
-
- @Override
- public boolean isQuoteRetouchEnable() {
- return quoteRetouchEnable;
- }
-
- @Override
- public void addMetaData(MetaData metaData) throws IOException {
- Column[] columns = ((TableMetaData) metaData).getColumns();
- String[] head = Stream.of(columns).map(Column::getColumnName).toArray(String[]::new);
- write(head);
- }
-
- private String compact(String[] row) {
- String quotationMarks = "\"";
- String dealNewlineSymbolMarks = "\n";
- StringBuilder rowBuilder = new StringBuilder();
- for (String value : row) {
- String decoratedValue = value;
- if (StringUtils.isNotBlank(value)) {
- if (quoteRetouchEnable) {
- decoratedValue = quotationMarks + value.replaceAll(quotationMarks, "") + quotationMarks;
- }
- decoratedValue = decoratedValue.replaceAll(dealNewlineSymbolMarks, " ");
- logger.debug("decorateValue with input: {} output: {} ", value, decoratedValue);
- }
- rowBuilder.append(decoratedValue).append(delimiter);
- }
- if (rowBuilder.length() > 0 && rowBuilder.toString().endsWith(delimiter)) {
- int index = rowBuilder.lastIndexOf(delimiter);
- rowBuilder.delete(index, index + delimiter.length());
- }
- rowBuilder.append("\n");
- if (logger.isDebugEnabled()) {
- logger.debug("delimiter:" + delimiter);
- }
- return rowBuilder.toString();
- }
-
- private void write(String[] row) throws IOException {
- String content = compact(row);
- if (buffer.length() + content.length() > 49500) {
- IOUtils.write(buffer.toString().getBytes(charset), outputStream);
- buffer.setLength(0);
- }
- buffer.append(content);
- }
-
- @Override
- public void addRecord(Record record) throws IOException {
- Object[] rows = ((TableRecord) record).row;
- String[] body =
- Stream.of(rows).map(dataType -> DataType.valueToString(dataType)).toArray(String[]::new);
- write(body);
- }
-
- @Override
- public void flush() throws IOException {
- IOUtils.write(buffer.toString().getBytes(charset), outputStream);
- buffer.setLength(0);
- }
-
- @Override
- public void close() throws IOException {
- flush();
- IOUtils.closeQuietly(outputStream);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Column.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Column.java
deleted file mode 100644
index 0d59805b07..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Column.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.domain;
-
-public final class Column {
- public String columnName;
- public DataType dataType;
- public String comment;
-
- public Column(String columnName, DataType dataType, String comment) {
- this.columnName = columnName;
- this.dataType = dataType;
- this.comment = comment;
- }
-
- public String getColumnName() {
- return columnName;
- }
-
- public DataType getDataType() {
- return dataType;
- }
-
- public String getComment() {
- return comment;
- }
-
- public Object[] toArray() {
- return new Object[] {columnName, dataType, comment};
- }
-
- @Override
- public String toString() {
- return "columnName:" + columnName + ",dataType:" + dataType + ",comment:" + comment;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/DataType.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/DataType.java
deleted file mode 100644
index 6808f693ec..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/DataType.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.domain;
-
-import java.math.BigDecimal;
-import java.sql.Date;
-import java.sql.Timestamp;
-import java.util.Optional;
-import java.util.regex.Pattern;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.linkis.storage.domain.Dolphin.LINKIS_NULL;
-
-public enum DataType {
- NullType("void", 0),
- StringType("string", 12),
- BooleanType("boolean", 16),
- TinyIntType("tinyint", -6),
- ShortIntType("short", 5),
- IntType("int", 4),
- LongType("long", -5),
- BigIntType("bigint", -5),
- FloatType("float", 6),
- DoubleType("double", 8),
- CharType("char", 1),
- VarcharType("varchar", 12),
- DateType("date", 91),
- TimestampType("timestamp", 93),
- BinaryType("binary", -2),
- DecimalType("decimal", 3),
- ArrayType("array", 2003),
- MapType("map", 2000),
- ListType("list", 2001),
- StructType("struct", 2002),
- BigDecimalType("bigdecimal", 3);
-
- private final String typeName;
- private final int javaSQLType;
-
- DataType(String typeName, int javaSQLType) {
- this.typeName = typeName;
- this.javaSQLType = javaSQLType;
- }
-
- private static Logger logger = LoggerFactory.getLogger(DataType.class);
-
- public static final String NULL_VALUE = "NULL";
- public static final String LOWCASE_NULL_VALUE = "null";
-
- // TODO Change to fine-grained regular expressions(改为精细化正则表达式)
- public static final Pattern DECIMAL_REGEX =
- Pattern.compile("^decimal\\(\\s*\\d*\\s*,\\s*\\d*\\s*\\)");
-
- public static final Pattern SHORT_REGEX = Pattern.compile("^short.*");
- public static final Pattern INT_REGEX = Pattern.compile("^int.*");
- public static final Pattern LONG_REGEX = Pattern.compile("^long.*");
- public static final Pattern BIGINT_REGEX = Pattern.compile("^bigint.*");
- public static final Pattern FLOAT_REGEX = Pattern.compile("^float.*");
- public static final Pattern DOUBLE_REGEX = Pattern.compile("^double.*");
-
- public static final Pattern VARCHAR_REGEX = Pattern.compile("^varchar.*");
- public static final Pattern CHAR_REGEX = Pattern.compile("^char.*");
-
- public static final Pattern ARRAY_REGEX = Pattern.compile("array.*");
-
- public static final Pattern MAP_REGEX = Pattern.compile("map.*");
-
- public static final Pattern LIST_REGEX = Pattern.compile("list.*");
-
- public static final Pattern STRUCT_REGEX = Pattern.compile("struct.*");
-
- public static DataType toDataType(String dataType) {
- if (dataType.equals("void") || dataType.equals("null")) {
- return DataType.NullType;
- } else if (dataType.equals("string")) {
- return DataType.StringType;
- } else if (dataType.equals("boolean")) {
- return DataType.BooleanType;
- } else if (SHORT_REGEX.matcher(dataType).matches()) {
- return DataType.ShortIntType;
- } else if (LONG_REGEX.matcher(dataType).matches()) {
- return DataType.LongType;
- } else if (BIGINT_REGEX.matcher(dataType).matches()) {
- return DataType.BigIntType;
- } else if (INT_REGEX.matcher(dataType).matches()
- || dataType.equals("integer")
- || dataType.equals("smallint")) {
- return DataType.IntType;
- } else if (FLOAT_REGEX.matcher(dataType).matches()) {
- return DataType.FloatType;
- } else if (DOUBLE_REGEX.matcher(dataType).matches()) {
- return DataType.DoubleType;
- } else if (VARCHAR_REGEX.matcher(dataType).matches()) {
- return DataType.VarcharType;
- } else if (CHAR_REGEX.matcher(dataType).matches()) {
- return DataType.CharType;
- } else if (dataType.equals("date")) {
- return DataType.DateType;
- } else if (dataType.equals("timestamp")) {
- return DataType.TimestampType;
- } else if (dataType.equals("binary")) {
- return DataType.BinaryType;
- } else if (dataType.equals("decimal") || DECIMAL_REGEX.matcher(dataType).matches()) {
- return DataType.DecimalType;
- } else if (ARRAY_REGEX.matcher(dataType).matches()) {
- return DataType.ArrayType;
- } else if (MAP_REGEX.matcher(dataType).matches()) {
- return DataType.MapType;
- } else if (LIST_REGEX.matcher(dataType).matches()) {
- return DataType.ListType;
- } else if (STRUCT_REGEX.matcher(dataType).matches()) {
- return DataType.StructType;
- } else {
- return DataType.StringType;
- }
- }
-
- public static Object toValue(DataType dataType, String value) {
-
- Object result = null;
- if (isLinkisNull(value)) {
- return result;
- }
- try {
- switch (dataType) {
- case NullType:
- result = null;
- break;
- case StringType:
- case CharType:
- case VarcharType:
- case StructType:
- case ListType:
- case ArrayType:
- case MapType:
- result = value;
- break;
- case BooleanType:
- result = isNumberNull(value) ? null : Boolean.valueOf(value);
- break;
- case ShortIntType:
- result = isNumberNull(value) ? null : Short.valueOf(value);
- break;
- case IntType:
- result = isNumberNull(value) ? null : Integer.valueOf(value);
- break;
- case LongType:
- case BigIntType:
- result = isNumberNull(value) ? null : Long.valueOf(value);
- break;
- case FloatType:
- result = isNumberNull(value) ? null : Float.valueOf(value);
- break;
- case DoubleType:
- result = isNumberNull(value) ? null : Double.valueOf(value);
- break;
- case DecimalType:
- result = isNumberNull(value) ? null : new BigDecimal(value);
- break;
- case DateType:
- result = isNumberNull(value) ? null : Date.valueOf(value);
- break;
- case TimestampType:
- result =
- isNumberNull(value)
- ? null
- : Optional.of(value)
- .map(Timestamp::valueOf)
- .map(Timestamp::toString)
- .map(s -> s.endsWith(".0") ? s.substring(0, s.length() - 2) : s)
- .orElse(null);
- break;
- case BinaryType:
- result = isNull(value) ? null : value.getBytes();
- break;
- default:
- result = value;
- }
- } catch (Exception e) {
- logger.debug("Failed to {} switch to dataType:", value, e);
- result = value;
- }
- return result;
- }
-
- public static boolean isLinkisNull(String value) {
- return value == null || value.equals(LINKIS_NULL);
- }
-
- public static boolean isNull(String value) {
- return value == null || value.equals(NULL_VALUE) || value.trim().equals("");
- }
-
- public static boolean isNumberNull(String value) {
- return value == null || value.equalsIgnoreCase(NULL_VALUE) || value.trim().equals("");
- }
-
- public static String valueToString(Object value) {
- if (value == null) {
- return LOWCASE_NULL_VALUE;
- } else if (value instanceof BigDecimal) {
- return ((BigDecimal) value).toPlainString();
- } else {
- return value.toString();
- }
- }
-
- public String getTypeName() {
- return typeName;
- }
-
- public int getJavaSQLType() {
- return javaSQLType;
- }
-
- @Override
- public String toString() {
- return typeName;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Dolphin.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Dolphin.java
deleted file mode 100644
index 35c71295e4..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Dolphin.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.domain;
-
-import org.apache.linkis.storage.exception.StorageWarnException;
-import org.apache.linkis.storage.utils.StorageConfiguration;
-import org.apache.linkis.storage.utils.StorageUtils;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
-import java.nio.charset.Charset;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.FAILED_TO_READ_INTEGER;
-
-public class Dolphin {
- private static final Logger logger = LoggerFactory.getLogger(Dolphin.class);
-
- public static final Charset CHAR_SET =
- Charset.forName(StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue());
- public static final String MAGIC = "dolphin";
-
- public static byte[] MAGIC_BYTES = new byte[0];
-
- static {
- try {
- MAGIC_BYTES = MAGIC.getBytes(StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue());
- } catch (UnsupportedEncodingException e) {
- logger.warn("Dolphin getBytes failed", e);
- }
- }
-
- public static final int MAGIC_LEN = MAGIC_BYTES.length;
-
- public static final String DOLPHIN_FILE_SUFFIX = ".dolphin";
-
- public static final String COL_SPLIT = ",";
- public static final byte[] COL_SPLIT_BYTES = COL_SPLIT.getBytes(Charset.forName("utf-8"));
- public static final int COL_SPLIT_LEN = COL_SPLIT_BYTES.length;
-
- public static final String NULL = "NULL";
- public static final byte[] NULL_BYTES = "NULL".getBytes(Charset.forName("utf-8"));
-
- public static final String LINKIS_NULL = "LINKIS_NULL";
- public static final byte[] LINKIS_NULL_BYTES = LINKIS_NULL.getBytes(Charset.forName("utf-8"));
-
- public static final int INT_LEN = 10;
-
- public static final int FILE_EMPTY = 31;
-
- public static byte[] getBytes(Object value) {
- return value.toString().getBytes(CHAR_SET);
- }
-
- /**
- * Convert a bytes array to a String content 将bytes数组转换为String内容
- *
- * @param bytes
- * @param start
- * @param len
- * @return
- */
- public static String getString(byte[] bytes, int start, int len) {
- return new String(bytes, start, len, Dolphin.CHAR_SET);
- }
-
- public static String toStringValue(String value) {
- if (LINKIS_NULL.equals(value)) {
- return NULL;
- } else {
- return value;
- }
- }
-
- /**
- * Read an integer value that converts the array to a byte of length 10 bytes
- * 读取整数值,该值为将数组转换为10字节长度的byte
- *
- * @param inputStream
- * @return
- * @throws IOException
- */
- public static int readInt(InputStream inputStream) throws IOException {
- byte[] bytes = new byte[INT_LEN + 1];
- if (StorageUtils.readBytes(inputStream, bytes, INT_LEN) != INT_LEN) {
- throw new StorageWarnException(
- FAILED_TO_READ_INTEGER.getErrorCode(), FAILED_TO_READ_INTEGER.getErrorDesc());
- }
- return Integer.parseInt(getString(bytes, 0, INT_LEN));
- }
-
- /**
- * Print integers at a fixed length(将整数按固定长度打印)
- *
- * @param value
- * @return
- */
- public static byte[] getIntBytes(int value) {
- String str = Integer.toString(value);
- StringBuilder res = new StringBuilder();
- for (int i = 0; i < INT_LEN - str.length(); i++) {
- res.append("0");
- }
- res.append(str);
- return Dolphin.getBytes(res.toString());
- }
-
- public static String getType(InputStream inputStream) throws IOException {
- byte[] bytes = new byte[100];
- int len = StorageUtils.readBytes(inputStream, bytes, Dolphin.MAGIC_LEN + INT_LEN);
- if (len == -1) return null;
- return getType(Dolphin.getString(bytes, 0, len));
- }
-
- public static String getType(String content) {
- if (content.length() < MAGIC.length() || !content.substring(0, MAGIC.length()).equals(MAGIC)) {
- throw new RuntimeException(
- "File header type must be dolphin, content: " + content + " is not");
- }
- return Integer.toString(
- Integer.parseInt(content.substring(MAGIC.length(), MAGIC.length() + INT_LEN)));
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/MethodEntity.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/MethodEntity.java
deleted file mode 100644
index c1e16e223a..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/MethodEntity.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.domain;
-
-/**
- * Engine unique Id(engine唯一的Id)
- *
- * <p>Fs type(fs类型)
- *
- * <p>Create a user to start the corresponding jvm user(创建用户为对应启动的jvm用户)
- *
- * <p>Proxy user(代理用户)
- *
- * <p>client Ip for whitelist control(ip用于白名单控制)
- *
- * <p>Method name called(调用的方法名)
- *
- * <p>Method parameter(方法参数)
- */
-public class MethodEntity {
- private long id;
- private String fsType;
- private String creatorUser;
- private String proxyUser;
- private String clientIp;
- private String methodName;
- private Object[] params;
-
- public MethodEntity(
- long id,
- String fsType,
- String creatorUser,
- String proxyUser,
- String clientIp,
- String methodName,
- Object[] params) {
- this.id = id;
- this.fsType = fsType;
- this.creatorUser = creatorUser;
- this.proxyUser = proxyUser;
- this.clientIp = clientIp;
- this.methodName = methodName;
- this.params = params;
- }
-
- public long getId() {
- return id;
- }
-
- public void setId(long id) {
- this.id = id;
- }
-
- public String getFsType() {
- return fsType;
- }
-
- public void setFsType(String fsType) {
- this.fsType = fsType;
- }
-
- public String getCreatorUser() {
- return creatorUser;
- }
-
- public void setCreatorUser(String creatorUser) {
- this.creatorUser = creatorUser;
- }
-
- public String getProxyUser() {
- return proxyUser;
- }
-
- public void setProxyUser(String proxyUser) {
- this.proxyUser = proxyUser;
- }
-
- public String getClientIp() {
- return clientIp;
- }
-
- public void setClientIp(String clientIp) {
- this.clientIp = clientIp;
- }
-
- public String getMethodName() {
- return methodName;
- }
-
- public void setMethodName(String methodName) {
- this.methodName = methodName;
- }
-
- public Object[] getParams() {
- return params;
- }
-
- public void setParams(Object[] params) {
- this.params = params;
- }
-
- @Override
- public String toString() {
- return "id:"
- + id
- + ", methodName:"
- + methodName
- + ", fsType:"
- + fsType
- + ", creatorUser:"
- + creatorUser
- + ", proxyUser:"
- + proxyUser
- + ", clientIp:"
- + clientIp;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/MethodEntitySerializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/MethodEntitySerializer.java
deleted file mode 100644
index 777b756a7f..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/MethodEntitySerializer.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.domain;
-
-import java.lang.reflect.Type;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-
-/**
- * Serialize MethodEntity to code 序列化MethodEntity为code
- *
- * <p>Serialized to code as a MethodEntity object 序列化为code为MethodEntity对象
- *
- * <p>Serialize a java object as a string 序列化java对象为字符串
- *
- * <p>Deserialize a string into a java object 将字符串解序列化为java对象
- */
-public class MethodEntitySerializer {
-
- private static final Gson gson =
- new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create();
-
- /**
- * Serialized to code as a MethodEntity object 序列化为code为MethodEntity对象
- *
- * @param code
- * @return
- */
- public static MethodEntity deserializer(String code) {
- return gson.fromJson(code, MethodEntity.class);
- }
-
- /**
- * Serialize MethodEntity to code 序列化MethodEntity为code
- *
- * @param methodEntity
- * @return
- */
- public static String serializer(MethodEntity methodEntity) {
- return gson.toJson(methodEntity);
- }
-
- /**
- * Serialize a java object as a string 序列化java对象为字符串
- *
- * @param value
- * @return
- */
- public static String serializerJavaObject(Object value) {
- return gson.toJson(value);
- }
-
- /**
- * Deserialize a string into a java object 将字符串解序列化为java对象
- *
- * @param json
- * @param classType
- * @param <T>
- * @return
- */
- public static <T> T deserializerToJavaObject(String json, Class<T> classType) {
- return gson.fromJson(json, classType);
- }
-
- public static <T> T deserializerToJavaObject(String json, Type oType) {
- return gson.fromJson(json, oType);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/errorcode/LinkisStorageErrorCodeSummary.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/errorcode/LinkisStorageErrorCodeSummary.java
index 9ca3425837..7187f72ce5 100644
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/errorcode/LinkisStorageErrorCodeSummary.java
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/errorcode/LinkisStorageErrorCodeSummary.java
@@ -37,6 +37,14 @@ public enum LinkisStorageErrorCodeSummary implements LinkisErrorCode {
52004,
"You must register IOMethodInterceptorCreator before you can use proxy mode.(必须先注册IOMethodInterceptorCreator,才能使用代理模式)"),
UNSUPPORTED_OPEN_FILE_TYPE(54001, "Unsupported open file type(不支持打开的文件类型)"),
+
+ RESULT_COL_LENGTH(52003, "Col value length {0} exceed limit {1}"),
+
+ RESULT_COLUMN_INDEX_OUT_OF_BOUNDS(52004, "Column index value {0} exceed limit {1}"),
+
+ RESULT_ROW_LENGTH(520034, "Row value length {0} exceed limit {1}"),
+
+ RESULT_COL_SIZE(520035, "Col size length {0} exceed limit {1}"),
INVALID_CUSTOM_PARAMETER(65000, "Invalid custom parameter(不合法的自定义参数)");
/** 错误码 */
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelAnalysisException.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelAnalysisException.java
deleted file mode 100644
index 87b988f240..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelAnalysisException.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.excel;
-
-class ExcelAnalysisException extends RuntimeException {
- public ExcelAnalysisException() {}
-
- public ExcelAnalysisException(String message) {
- super(message);
- }
-
- public ExcelAnalysisException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public ExcelAnalysisException(Throwable cause) {
- super(cause);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelFsWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelFsWriter.java
deleted file mode 100644
index cd3969e048..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelFsWriter.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.excel;
-
-import org.apache.linkis.common.io.FsWriter;
-
-import java.io.OutputStream;
-
-public abstract class ExcelFsWriter extends FsWriter {
- public abstract String getCharset();
-
- public abstract String getSheetName();
-
- public abstract String getDateFormat();
-
- public abstract boolean isAutoFormat();
-
- public static ExcelFsWriter getExcelFsWriter(
- String charset,
- String sheetName,
- String dateFormat,
- OutputStream outputStream,
- boolean autoFormat) {
- return new StorageExcelWriter(charset, sheetName, dateFormat, outputStream, autoFormat);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelXlsReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelXlsReader.java
deleted file mode 100644
index 98df7421f3..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelXlsReader.java
+++ /dev/null
@@ -1,278 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.excel;
-
-import org.apache.poi.hssf.eventusermodel.*;
-import org.apache.poi.hssf.eventusermodel.EventWorkbookBuilder.SheetRecordCollectingListener;
-import org.apache.poi.hssf.eventusermodel.dummyrecord.LastCellOfRowDummyRecord;
-import org.apache.poi.hssf.eventusermodel.dummyrecord.MissingCellDummyRecord;
-import org.apache.poi.hssf.model.HSSFFormulaParser;
-import org.apache.poi.hssf.record.*;
-import org.apache.poi.hssf.usermodel.HSSFWorkbook;
-import org.apache.poi.poifs.filesystem.POIFSFileSystem;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ExcelXlsReader implements HSSFListener {
- private static final Logger logger = LoggerFactory.getLogger(ExcelXlsReader.class);
-
- private int minColumns = -1;
-
- private POIFSFileSystem fs;
-
- private InputStream inputStream;
-
- private int lastRowNumber;
-
- private int lastColumnNumber;
-
- /** Should we output the formula, or the value it has? */
- private boolean outputFormulaValues = true;
-
- /** For parsing Formulas */
- private SheetRecordCollectingListener workbookBuildingListener;
-
- // excel2003Workbook(工作薄)
- private HSSFWorkbook stubWorkbook;
-
- // Records we pick up as we process
- private SSTRecord sstRecord;
-
- private FormatTrackingHSSFListener formatListener;
-
- // Table index(表索引)
- private int sheetIndex = -1;
-
- private BoundSheetRecord[] orderedBSRs;
-
- @SuppressWarnings("unchecked")
- private ArrayList boundSheetRecords = new ArrayList();
-
- // For handling formulas with string results
- private int nextRow;
-
- private int nextColumn;
-
- private boolean outputNextStringRecord;
-
- // Current line(当前行)
- private int curRow = 0;
-
- // a container that stores row records(存储行记录的容器)
- private List<String> rowlist = new ArrayList<String>();
-
- @SuppressWarnings("unused")
- private String sheetName;
-
- private IExcelRowDeal excelRowDeal;
-
- public void init(IExcelRowDeal excelRowDeal, InputStream inputStream) {
- this.excelRowDeal = excelRowDeal;
- this.inputStream = inputStream;
- }
-
- /**
- * Traverse all the sheets under excel 遍历excel下所有的sheet
- *
- * @throws IOException
- */
- public void process() throws IOException {
- this.fs = new POIFSFileSystem(this.inputStream);
- MissingRecordAwareHSSFListener listener = new MissingRecordAwareHSSFListener(this);
- formatListener = new FormatTrackingHSSFListener(listener);
- HSSFEventFactory factory = new HSSFEventFactory();
- HSSFRequest request = new HSSFRequest();
- if (outputFormulaValues) {
- request.addListenerForAllRecords(formatListener);
- } else {
- workbookBuildingListener = new SheetRecordCollectingListener(formatListener);
- request.addListenerForAllRecords(workbookBuildingListener);
- }
- factory.processWorkbookEvents(request, fs);
- }
-
- /** HSSFListener listener method, processing Record HSSFListener 监听方法,处理 Record */
- @Override
- @SuppressWarnings("unchecked")
- public void processRecord(Record record) {
- int thisRow = -1;
- int thisColumn = -1;
- String thisStr = null;
- String value = null;
- switch (record.getSid()) {
- case BoundSheetRecord.sid:
- boundSheetRecords.add(record);
- break;
- case BOFRecord.sid:
- BOFRecord br = (BOFRecord) record;
- if (br.getType() == BOFRecord.TYPE_WORKSHEET) {
- // Create a child workbook if needed(如果有需要,则建立子工作薄)
- if (workbookBuildingListener != null && stubWorkbook == null) {
- stubWorkbook = workbookBuildingListener.getStubHSSFWorkbook();
- }
-
- sheetIndex++;
- if (orderedBSRs == null) {
- orderedBSRs = BoundSheetRecord.orderByBofPosition(boundSheetRecords);
- }
- sheetName = orderedBSRs[sheetIndex].getSheetname();
- }
- break;
-
- case SSTRecord.sid:
- sstRecord = (SSTRecord) record;
- break;
-
- case BlankRecord.sid:
- BlankRecord brec = (BlankRecord) record;
- thisRow = brec.getRow();
- thisColumn = brec.getColumn();
- thisStr = "";
- rowlist.add(thisColumn, thisStr);
- break;
- case BoolErrRecord.sid: // Cell is boolean(单元格为布尔类型)
- BoolErrRecord berec = (BoolErrRecord) record;
- thisRow = berec.getRow();
- thisColumn = berec.getColumn();
- thisStr = berec.getBooleanValue() + "";
- rowlist.add(thisColumn, thisStr);
- break;
-
- case FormulaRecord.sid: // Cell is a formula type(单元格为公式类型)
- FormulaRecord frec = (FormulaRecord) record;
- thisRow = frec.getRow();
- thisColumn = frec.getColumn();
- if (outputFormulaValues) {
- if (Double.isNaN(frec.getValue())) {
- // Formula result is a string
- // This is stored in the next record
- outputNextStringRecord = true;
- nextRow = frec.getRow();
- nextColumn = frec.getColumn();
- } else {
- thisStr = formatListener.formatNumberDateCell(frec);
- }
- } else {
- thisStr =
- '"'
- + HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression())
- + '"';
- }
- rowlist.add(thisColumn, thisStr);
- break;
- case StringRecord.sid: // a string of formulas in a cell(单元格中公式的字符串)
- if (outputNextStringRecord) {
- // String for formula
- StringRecord srec = (StringRecord) record;
- thisStr = srec.getString();
- thisRow = nextRow;
- thisColumn = nextColumn;
- outputNextStringRecord = false;
- }
- break;
- case LabelRecord.sid:
- LabelRecord lrec = (LabelRecord) record;
- curRow = thisRow = lrec.getRow();
- thisColumn = lrec.getColumn();
- value = lrec.getValue().trim();
- value = value.equals("") ? " " : value;
- this.rowlist.add(thisColumn, value);
- break;
- case LabelSSTRecord.sid: // Cell is a string type(单元格为字符串类型)
- LabelSSTRecord lsrec = (LabelSSTRecord) record;
- curRow = thisRow = lsrec.getRow();
- thisColumn = lsrec.getColumn();
- if (sstRecord == null) {
- rowlist.add(thisColumn, " ");
- } else {
- value = sstRecord.getString(lsrec.getSSTIndex()).toString().trim();
- value = value.equals("") ? " " : value;
- rowlist.add(thisColumn, value);
- }
- break;
- case NumberRecord.sid: // Cell is a numeric type(单元格为数字类型)
- NumberRecord numrec = (NumberRecord) record;
- curRow = thisRow = numrec.getRow();
- thisColumn = numrec.getColumn();
- value = formatListener.formatNumberDateCell(numrec).trim();
- value = value.equals("") ? "0" : value;
- // Add column values to the container(向容器加入列值)
- rowlist.add(thisColumn, value);
- break;
- default:
- break;
- }
-
- // Encountered a new line of operations(遇到新行的操作)(
- if (thisRow != -1 && thisRow != lastRowNumber) {
- lastColumnNumber = -1;
- }
-
- // Null operation(空值的操作)
- if (record instanceof MissingCellDummyRecord) {
- MissingCellDummyRecord mc = (MissingCellDummyRecord) record;
- curRow = thisRow = mc.getRow();
- thisColumn = mc.getColumn();
- rowlist.add(thisColumn, " ");
- }
-
- // Update row and column values(更新行和列的值)
- if (thisRow > -1) lastRowNumber = thisRow;
- if (thisColumn > -1) lastColumnNumber = thisColumn;
-
- // End of line operation(行结束时的操作)
- if (record instanceof LastCellOfRowDummyRecord) {
- if (minColumns > 0) {
- // Column value is re-empted(列值重新置空)
- if (lastColumnNumber == -1) {
- lastColumnNumber = 0;
- }
- }
- lastColumnNumber = -1;
-
- // At the end of each line, the dealRow() method(每行结束时, dealRow() 方法)
- excelRowDeal.dealRow(orderedBSRs, sheetIndex, curRow, rowlist);
- // Empty container(清空容器)
- rowlist.clear();
- }
- }
-
- public void close() {
- try {
- if (fs != null) {
- fs.close();
- }
- } catch (Exception e) {
- logger.info("ExcelXlsReader fs closed failed", e);
- }
-
- try {
- if (inputStream != null) {
- inputStream.close();
- }
- } catch (IOException e) {
- logger.info("ExcelXlsReader inputStream closed failed", e);
- }
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/FirstRowDeal.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/FirstRowDeal.java
deleted file mode 100644
index 6924a3ebb9..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/FirstRowDeal.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.excel;
-
-import org.apache.poi.hssf.record.BoundSheetRecord;
-
-import java.util.ArrayList;
-import java.util.List;
-
-class FirstRowDeal implements IExcelRowDeal {
-
- private List<String> sheetNames = new ArrayList<>();
- private List<String> row;
-
- public List<String> getSheetNames() {
- return sheetNames;
- }
-
- public void setSheetNames(List<String> sheetNames) {
- this.sheetNames = sheetNames;
- }
-
- public List<String> getRow() {
- return row;
- }
-
- public void setRow(List<String> row) {
- this.row = row;
- }
-
- @Override
- public void dealRow(
- BoundSheetRecord[] orderedBSRs, int sheetIndex, int curRow, List<String> rowlist) {
- for (BoundSheetRecord record : orderedBSRs) {
- sheetNames.add(record.getSheetname());
- }
- row = rowlist;
- throw new ExcelAnalysisException("Finished to deal first row");
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/IExcelRowDeal.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/IExcelRowDeal.java
deleted file mode 100644
index 4051041747..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/IExcelRowDeal.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.excel;
-
-import org.apache.poi.hssf.record.BoundSheetRecord;
-
-import java.util.List;
-
-interface IExcelRowDeal {
- void dealRow(BoundSheetRecord[] orderedBSRs, int sheetIndex, int curRow, List<String> rowlist);
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/RowToCsvDeal.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/RowToCsvDeal.java
deleted file mode 100644
index 7deccfb92a..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/RowToCsvDeal.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.excel;
-
-import org.apache.poi.hssf.record.BoundSheetRecord;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.List;
-import java.util.Map;
-
-class RowToCsvDeal implements IExcelRowDeal {
-
- private Map<String, Object> params;
- private List<String> sheetNames;
- private OutputStream outputStream;
- private Boolean hasHeader;
- private Boolean fisrtRow = true;
-
- public void init(Boolean hasHeader, List<String> sheetNames, OutputStream outputStream) {
- this.hasHeader = hasHeader;
- this.sheetNames = sheetNames;
- this.outputStream = outputStream;
- }
-
- @Override
- public void dealRow(
- BoundSheetRecord[] orderedBSRs, int sheetIndex, int curRow, List<String> rowlist) {
- String sheetName = orderedBSRs[sheetIndex].getSheetname();
- if (sheetNames == null || sheetNames.isEmpty() || sheetNames.contains(sheetName)) {
- if (!(curRow == 0 && hasHeader)) {
- try {
- if (fisrtRow) {
- fisrtRow = false;
- } else {
- outputStream.write("\n".getBytes());
- }
- int len = rowlist.size();
- for (int i = 0; i < len; i++) {
- outputStream.write(rowlist.get(i).replaceAll("\n|\t", " ").getBytes("utf-8"));
- if (i < len - 1) {
- outputStream.write("\t".getBytes());
- }
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/StorageExcelWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/StorageExcelWriter.java
deleted file mode 100644
index 6b2a98c72b..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/StorageExcelWriter.java
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.excel;
-
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.storage.domain.Column;
-import org.apache.linkis.storage.domain.DataType;
-import org.apache.linkis.storage.resultset.table.TableMetaData;
-import org.apache.linkis.storage.resultset.table.TableRecord;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.poi.ss.usermodel.*;
-import org.apache.poi.xssf.streaming.SXSSFSheet;
-import org.apache.poi.xssf.streaming.SXSSFWorkbook;
-
-import java.io.*;
-import java.math.BigDecimal;
-import java.util.*;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class StorageExcelWriter extends ExcelFsWriter {
-
- private static Logger logger = LoggerFactory.getLogger(StorageExcelWriter.class);
-
- private String charset;
- private String sheetName;
- private String dateFormat;
- private OutputStream outputStream;
- private boolean autoFormat;
- protected SXSSFWorkbook workBook;
- protected SXSSFSheet sheet;
- private DataFormat format;
- protected DataType[] types;
- protected int rowPoint;
- protected int columnCounter;
- protected Map<String, CellStyle> styles = new HashMap<>();
- private boolean isFlush = true;
- private ByteArrayOutputStream os = new ByteArrayOutputStream();
- private ByteArrayInputStream is;
-
- public StorageExcelWriter(
- String charset,
- String sheetName,
- String dateFormat,
- OutputStream outputStream,
- boolean autoFormat) {
- this.charset = charset;
- this.sheetName = sheetName;
- this.dateFormat = dateFormat;
- this.outputStream = outputStream;
- this.autoFormat = autoFormat;
- }
-
- public void init() {
- workBook = new SXSSFWorkbook();
- sheet = workBook.createSheet(sheetName);
- }
-
- public CellStyle getDefaultHeadStyle() {
- Font headerFont = workBook.createFont();
- headerFont.setBold(true);
- headerFont.setFontHeightInPoints((short) 14);
- headerFont.setColor(IndexedColors.RED.getIndex());
- CellStyle headerCellStyle = workBook.createCellStyle();
- headerCellStyle.setFont(headerFont);
- return headerCellStyle;
- }
-
- public Workbook getWorkBook() {
- // 自适应列宽
- sheet.trackAllColumnsForAutoSizing();
- for (int elem = 0; elem <= columnCounter; elem++) {
- sheet.autoSizeColumn(elem);
- }
- return workBook;
- }
-
- public CellStyle createCellStyle(DataType dataType) {
- CellStyle style = workBook.createCellStyle();
- format = workBook.createDataFormat();
- style.setDataFormat(format.getFormat("@"));
-
- if (autoFormat) {
- switch (dataType) {
- case StringType:
- case CharType:
- case VarcharType:
- style.setDataFormat(format.getFormat("@"));
- break;
- case TinyIntType:
- case ShortIntType:
- case IntType:
- style.setDataFormat(format.getFormat("#"));
- break;
- case LongType:
- case BigIntType:
- style.setDataFormat(format.getFormat("#.##E+00"));
- break;
- case FloatType:
- style.setDataFormat(format.getFormat("#.0000000000"));
- break;
- case DoubleType:
- style.setDataFormat(format.getFormat("#.0000000000"));
- break;
- case DateType:
- case TimestampType:
- style.setDataFormat(format.getFormat("m/d/yy h:mm"));
- break;
- case DecimalType:
- case BigDecimalType:
- style.setDataFormat(format.getFormat("#.000000000"));
- break;
- default:
- style.setDataFormat(format.getFormat("@"));
- }
- }
- return style;
- }
-
- public CellStyle getCellStyle(DataType dataType) {
- CellStyle style = styles.get(dataType.getTypeName());
- if (style == null) {
- CellStyle newStyle = createCellStyle(dataType);
- styles.put(dataType.getTypeName(), newStyle);
- return newStyle;
- } else {
- return style;
- }
- }
-
- @Override
- public void addMetaData(MetaData metaData) throws IOException {
- init();
- Row tableHead = sheet.createRow(0);
- Column[] columns = ((TableMetaData) metaData).getColumns();
- List<DataType> columnType = new ArrayList<>();
- for (int i = 0; i < columns.length; i++) {
- Cell headCell = tableHead.createCell(columnCounter);
- headCell.setCellValue(columns[i].getColumnName());
- headCell.setCellStyle(getDefaultHeadStyle());
- columnType.add(columns[i].getDataType());
- columnCounter++;
- }
- types = columnType.toArray(new DataType[0]);
- rowPoint++;
- }
-
- @Override
- public void addRecord(Record record) throws IOException {
- // TODO: 是否需要替换null值
- Row tableBody = sheet.createRow(rowPoint);
- int colunmPoint = 0;
- Object[] excelRecord = ((TableRecord) record).row;
- for (Object elem : excelRecord) {
- Cell cell = tableBody.createCell(colunmPoint);
- DataType dataType = types[colunmPoint];
- if (autoFormat) {
- setCellTypeValue(dataType, elem, cell);
- } else {
- cell.setCellValue(DataType.valueToString(elem));
- }
- cell.setCellStyle(getCellStyle(dataType));
- colunmPoint++;
- }
- rowPoint++;
- }
-
- private void setCellTypeValue(DataType dataType, Object elem, Cell cell) {
- if (null == elem) return;
-
- try {
- switch (dataType) {
- case StringType:
- case CharType:
- case VarcharType:
- cell.setCellValue(DataType.valueToString(elem));
- break;
- case TinyIntType:
- case ShortIntType:
- case IntType:
- cell.setCellValue(Integer.valueOf(elem.toString()));
- break;
- case LongType:
- case BigIntType:
- cell.setCellValue(Long.valueOf(elem.toString()));
- break;
- case FloatType:
- cell.setCellValue(Float.valueOf(elem.toString()));
- break;
- case DoubleType:
- doubleCheck(elem.toString());
- cell.setCellValue(Double.valueOf(elem.toString()));
- break;
- case DateType:
- case TimestampType:
- cell.setCellValue(getDate(elem));
- break;
- case DecimalType:
- case BigDecimalType:
- doubleCheck(DataType.valueToString(elem));
- cell.setCellValue(Double.valueOf(DataType.valueToString(elem)));
- break;
- default:
- cell.setCellValue(DataType.valueToString(elem));
- }
- } catch (Exception e) {
- cell.setCellValue(DataType.valueToString(elem));
- }
- }
-
- private Date getDate(Object value) {
- if (value instanceof Date) {
- return (Date) value;
- } else {
- throw new NumberFormatException(
- "Value "
- + value
- + " with class : "
- + value.getClass().getName()
- + " is not a valid type of Date.");
- }
- }
-
- /**
- * Check whether the double exceeds the number of digits, which will affect the data accuracy
- *
- * @param elemValue
- */
- private void doubleCheck(String elemValue) {
- BigDecimal value = new BigDecimal(elemValue).stripTrailingZeros();
- if ((value.precision() - value.scale()) > 15) {
- throw new NumberFormatException(
- "Value " + elemValue + " error : This data exceeds 15 significant digits.");
- }
- }
-
- @Override
- public void flush() {
- try {
- getWorkBook().write(os);
- } catch (IOException e) {
- logger.warn("flush fail", e);
- }
- byte[] content = os.toByteArray();
- is = new ByteArrayInputStream(content);
- byte[] buffer = new byte[1024];
- int bytesRead = 0;
- while (isFlush) {
- try {
- bytesRead = is.read(buffer, 0, 1024);
- if (bytesRead == -1) {
- isFlush = false;
- } else {
- outputStream.write(buffer, 0, bytesRead);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- @Override
- public void close() {
- if (isFlush) {
- flush();
- }
- IOUtils.closeQuietly(outputStream);
- IOUtils.closeQuietly(is);
- IOUtils.closeQuietly(os);
- IOUtils.closeQuietly(workBook);
- }
-
- @Override
- public String getCharset() {
- return this.charset;
- }
-
- @Override
- public String getSheetName() {
- return this.sheetName;
- }
-
- @Override
- public String getDateFormat() {
- return this.dateFormat;
- }
-
- @Override
- public boolean isAutoFormat() {
- return this.autoFormat;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/StorageMultiExcelWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/StorageMultiExcelWriter.java
deleted file mode 100644
index a028397bd6..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/StorageMultiExcelWriter.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.excel;
-
-import org.apache.poi.xssf.streaming.SXSSFWorkbook;
-
-import java.io.OutputStream;
-
-public class StorageMultiExcelWriter extends StorageExcelWriter {
-
- private int sheetIndex = 0;
-
- public StorageMultiExcelWriter(OutputStream outputStream, boolean autoFormat) {
- super(null, null, null, outputStream, autoFormat);
- }
-
- @Override
- public void init() {
- if (workBook == null) {
- workBook = new SXSSFWorkbook();
- }
- // 1.让表自适应列宽
- if (sheet != null) {
- sheet.trackAllColumnsForAutoSizing();
- for (int i = 0; i <= columnCounter; i++) {
- sheet.autoSizeColumn(i);
- }
- }
- // 2.重置参数
- // 2.1 columnCounter 归0
- columnCounter = 0;
- // 2.2 创建新sheet
- sheet = workBook.createSheet("resultset" + sheetIndex);
- // 2.3 sheetIndex自增
- sheetIndex++;
- // 2.4 types 置空
- types = null;
- // 2.5 rowPoint 归0 记录行数
- rowPoint = 0;
- // 2.6 styles 清空
- styles.clear();
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java
index 246fb79bc3..39d89c3d96 100644
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java
@@ -21,12 +21,17 @@ import org.apache.linkis.storage.utils.StorageUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.poi.hssf.usermodel.HSSFWorkbook;
+import org.apache.poi.openxml4j.util.ZipSecureFile;
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,7 +66,7 @@ public class XlsUtils {
throws Exception {
String hdfsPath =
"/tmp/" + StorageUtils.getJvmUser() + "/" + System.currentTimeMillis() + ".csv";
- LOG.info("The excel to csv with hdfsPath:" + hdfsPath);
+ LOG.info("The excel to csv with hdfs path:" + hdfsPath);
ExcelXlsReader xlsReader = new ExcelXlsReader();
RowToCsvDeal rowToCsvDeal = new RowToCsvDeal();
OutputStream out = null;
@@ -81,4 +86,44 @@ public class XlsUtils {
}
return hdfsPath;
}
+
+ public static Map<String, List<Map<String, String>>> getSheetsInfo(
+ InputStream inputStream, Boolean hasHeader) {
+ // use xls file
+ Workbook workbook = null;
+ try {
+ // 压缩膨胀比率,处理excel行或者列过多的情况,不能设置再小了,会导致内存过大
+ ZipSecureFile.setMinInflateRatio(0.005);
+ workbook = new HSSFWorkbook(inputStream);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ // 使用完最后需要还原
+ ZipSecureFile.setMinInflateRatio(0.01);
+ }
+ Map<String, List<Map<String, String>>> res = new LinkedHashMap<>(workbook.getNumberOfSheets());
+ // foreach Sheet
+ for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
+ Sheet sheet = workbook.getSheetAt(i);
+
+ List<Map<String, String>> rowList = new ArrayList<>();
+
+ // get first row as column name
+ Row headerRow = sheet.getRow(0);
+
+ // foreach column
+ for (int j = 0; j < headerRow.getPhysicalNumberOfCells(); j++) {
+ Map<String, String> sheetMap = new LinkedHashMap<>();
+ Cell cell = headerRow.getCell(j);
+ if (hasHeader) {
+ sheetMap.put(cell.getStringCellValue(), "string");
+ } else {
+ sheetMap.put("col_" + (j + 1), "string");
+ }
+ rowList.add(sheetMap);
+ }
+ res.put(sheet.getSheetName(), rowList);
+ }
+ return res;
+ }
}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsxUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsxUtils.java
index 7cbf579a9d..c2418f4c33 100644
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsxUtils.java
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsxUtils.java
@@ -23,10 +23,9 @@ import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import java.io.File;
+import java.io.IOException;
import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
+import java.util.*;
import com.github.pjfanning.xlsx.StreamingReader;
@@ -79,4 +78,54 @@ public class XlsxUtils {
}
}
}
+
+ public static Map<String, List<Map<String, String>>> getAllSheetInfo(
+ InputStream inputStream, File file, Boolean hasHeader) throws IOException {
+ try {
+ Workbook wb = null;
+ if (inputStream != null) {
+ wb =
+ StreamingReader.builder()
+ // number of rows to keep in memory (defaults to 10)
+ .rowCacheSize(2)
+ .open(inputStream);
+ } else {
+ wb =
+ StreamingReader.builder()
+ // number of rows to keep in memory (defaults to 10)
+ .rowCacheSize(2)
+ .open(file);
+ }
+ Map<String, List<Map<String, String>>> res = new LinkedHashMap<>(wb.getNumberOfSheets());
+ for (Sheet sheet : wb) {
+ Iterator<Row> iterator = sheet.iterator();
+ Row row = null;
+ while (iterator.hasNext() && row == null) {
+ row = iterator.next();
+ }
+ List<Map<String, String>> rowList = new ArrayList<>();
+ if (row == null) {
+ res.put(sheet.getSheetName(), rowList);
+ continue;
+ }
+ int cellIdx = 0;
+ for (Cell cell : row) {
+ Map<String, String> item = new LinkedHashMap<>();
+ if (hasHeader) {
+ item.put(cell.getStringCellValue(), "string");
+ } else {
+ item.put("col_" + (cellIdx + 1), "string");
+ }
+ cellIdx++;
+ rowList.add(item);
+ }
+ res.put(sheet.getSheetName(), rowList);
+ }
+ return res;
+ } finally {
+ if (inputStream != null) {
+ inputStream.close();
+ }
+ }
+ }
}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/ColLengthExceedException.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/ColLengthExceedException.java
new file mode 100644
index 0000000000..955a8e1d60
--- /dev/null
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/ColLengthExceedException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.storage.exception;
+
+public class ColLengthExceedException extends StorageWarnException {
+
+ public ColLengthExceedException(int errCode, String desc) {
+ super(errCode, desc);
+ }
+
+ public ColLengthExceedException(
+ int errCode, String desc, String ip, int port, String serviceKind) {
+ super(errCode, desc, ip, port, serviceKind);
+ }
+
+ public ColLengthExceedException(int errCode, String desc, Throwable t) {
+ super(errCode, desc, t);
+ }
+}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/ColumnIndexExceedException.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/ColumnIndexExceedException.java
new file mode 100644
index 0000000000..969b19d20b
--- /dev/null
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/ColumnIndexExceedException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.storage.exception;
+
+public class ColumnIndexExceedException extends StorageWarnException {
+
+ public ColumnIndexExceedException(int errCode, String desc) {
+ super(errCode, desc);
+ }
+
+ public ColumnIndexExceedException(
+ int errCode, String desc, String ip, int port, String serviceKind) {
+ super(errCode, desc, ip, port, serviceKind);
+ }
+
+ public ColumnIndexExceedException(int errCode, String desc, Throwable t) {
+ super(errCode, desc, t);
+ }
+}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java
index 7a8452141b..ae66c1cf99 100644
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java
@@ -21,10 +21,8 @@ public enum StorageErrorCode {
/** */
FS_NOT_INIT(53001, "please init first"),
-
INCONSISTENT_DATA(53001, "Inconsistent row data read,read %s,need rowLen %s"),
FS_OOM(53002, "OOM occurred while reading the file"),
-
FS_ERROR(53003, "Failed to operation fs"),
READ_PARQUET_FAILED(53004, "Failed to read parquet file"),
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildHDFSFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildHDFSFileSystem.java
index 8103c6f3de..9f53a6249b 100644
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildHDFSFileSystem.java
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildHDFSFileSystem.java
@@ -21,7 +21,7 @@ import org.apache.linkis.common.io.Fs;
import org.apache.linkis.storage.factory.BuildFactory;
import org.apache.linkis.storage.fs.FileSystem;
import org.apache.linkis.storage.fs.impl.HDFSFileSystem;
-import org.apache.linkis.storage.io.IOMethodInterceptorFactory;
+import org.apache.linkis.storage.io.IOMethodInterceptorCreator$;
import org.apache.linkis.storage.utils.StorageUtils;
import org.springframework.cglib.proxy.Enhancer;
@@ -46,7 +46,7 @@ public class BuildHDFSFileSystem implements BuildFactory {
// TODO Agent user(代理的用户)
Enhancer enhancer = new Enhancer();
enhancer.setSuperclass(HDFSFileSystem.class.getSuperclass());
- enhancer.setCallback(IOMethodInterceptorFactory.getIOMethodInterceptor(fsName()));
+ enhancer.setCallback(IOMethodInterceptorCreator$.MODULE$.getIOMethodInterceptor(fsName()));
fs = (FileSystem) enhancer.create();
}
fs.setUser(proxyUser);
@@ -63,6 +63,6 @@ public class BuildHDFSFileSystem implements BuildFactory {
@Override
public String fsName() {
- return StorageUtils.HDFS;
+ return StorageUtils.HDFS();
}
}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildLocalFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildLocalFileSystem.java
index bcd61c5735..ef88cec36d 100644
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildLocalFileSystem.java
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildLocalFileSystem.java
@@ -21,7 +21,7 @@ import org.apache.linkis.common.io.Fs;
import org.apache.linkis.storage.factory.BuildFactory;
import org.apache.linkis.storage.fs.FileSystem;
import org.apache.linkis.storage.fs.impl.LocalFileSystem;
-import org.apache.linkis.storage.io.IOMethodInterceptorFactory;
+import org.apache.linkis.storage.io.IOMethodInterceptorCreator$;
import org.apache.linkis.storage.utils.StorageConfiguration;
import org.apache.linkis.storage.utils.StorageUtils;
@@ -34,7 +34,7 @@ public class BuildLocalFileSystem implements BuildFactory {
FileSystem fs = null;
if (StorageUtils.isIOProxy()) {
if (user.equals(proxyUser)) {
- if ((Boolean) StorageConfiguration.IS_SHARE_NODE.getValue()) {
+ if ((Boolean) StorageConfiguration.IS_SHARE_NODE().getValue()) {
fs = new LocalFileSystem();
} else {
fs = getProxyFs();
@@ -58,12 +58,12 @@ public class BuildLocalFileSystem implements BuildFactory {
private FileSystem getProxyFs() {
Enhancer enhancer = new Enhancer();
enhancer.setSuperclass(LocalFileSystem.class.getSuperclass());
- enhancer.setCallback(IOMethodInterceptorFactory.getIOMethodInterceptor(fsName()));
+ enhancer.setCallback(IOMethodInterceptorCreator$.MODULE$.getIOMethodInterceptor(fsName()));
return (FileSystem) enhancer.create();
}
@Override
public String fsName() {
- return StorageUtils.FILE;
+ return StorageUtils.FILE();
}
}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildOSSSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildOSSSystem.java
deleted file mode 100644
index ba1bd7abef..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildOSSSystem.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.factory.impl;
-
-import org.apache.linkis.common.io.Fs;
-import org.apache.linkis.storage.factory.BuildFactory;
-import org.apache.linkis.storage.fs.impl.OSSFileSystem;
-import org.apache.linkis.storage.utils.StorageUtils;
-
-import java.io.IOException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class BuildOSSSystem implements BuildFactory {
-
- private static final Logger LOG = LoggerFactory.getLogger(BuildOSSSystem.class);
-
- /**
- * get file system
- *
- * @param user
- * @param proxyUser
- * @return
- */
- @Override
- public Fs getFs(String user, String proxyUser) {
- OSSFileSystem fs = new OSSFileSystem();
- try {
- fs.init(null);
- } catch (IOException e) {
- LOG.warn("get file system failed", e);
- }
- fs.setUser(user);
- return fs;
- }
-
- @Override
- public Fs getFs(String user, String proxyUser, String label) {
- OSSFileSystem fs = new OSSFileSystem();
- try {
- fs.init(null);
- } catch (IOException e) {
- LOG.warn("get file system failed", e);
- }
- fs.setUser(proxyUser);
- fs.setLabel(label);
- return fs;
- }
-
- @Override
- public String fsName() {
- return StorageUtils.OSS;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildS3FileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildS3FileSystem.java
deleted file mode 100644
index 44082e5898..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildS3FileSystem.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.factory.impl;
-
-import org.apache.linkis.common.io.Fs;
-import org.apache.linkis.storage.factory.BuildFactory;
-import org.apache.linkis.storage.fs.impl.S3FileSystem;
-import org.apache.linkis.storage.utils.StorageUtils;
-
-import java.io.IOException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class BuildS3FileSystem implements BuildFactory {
- private static final Logger LOG = LoggerFactory.getLogger(BuildS3FileSystem.class);
-
- @Override
- public Fs getFs(String user, String proxyUser) {
- S3FileSystem fs = new S3FileSystem();
- try {
- fs.init(null);
- } catch (IOException e) {
- LOG.warn("get file system failed", e);
- }
- fs.setUser(user);
- return fs;
- }
-
- @Override
- public Fs getFs(String user, String proxyUser, String label) {
- S3FileSystem fs = new S3FileSystem();
- try {
- fs.init(null);
- } catch (IOException e) {
- LOG.warn("get file system failed", e);
- }
- fs.setUser(user);
- fs.setLabel(label);
- return fs;
- }
-
- @Override
- public String fsName() {
- return StorageUtils.S3;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java
index 3067383b6c..ac828267bf 100644
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java
@@ -51,6 +51,10 @@ public abstract class FileSystem implements Fs {
public abstract long getUsableSpace(FsPath dest) throws IOException;
+ public abstract long getLength(FsPath dest) throws IOException;
+
+ public abstract String checkSum(FsPath dest) throws IOException;
+
public abstract boolean canExecute(FsPath dest) throws IOException;
public abstract boolean setOwner(FsPath dest, String user, String group) throws IOException;
@@ -99,7 +103,7 @@ public abstract class FileSystem implements Fs {
} else {
parentPath = path.substring(0, path.lastIndexOf("/"));
}
- LOG.info("Get Parent Path:" + parentPath);
+ LOG.info("Get parent path:" + parentPath);
return new FsPath(parentPath);
}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java
index f40d75c040..c4f4814149 100644
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java
@@ -31,9 +31,7 @@ import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
@@ -82,7 +80,7 @@ public class HDFSFileSystem extends FileSystem {
@Override
public boolean canExecute(FsPath dest) throws IOException {
- return canAccess(dest, FsAction.EXECUTE);
+ return canAccess(dest, FsAction.EXECUTE, this.user);
}
@Override
@@ -162,7 +160,8 @@ public class HDFSFileSystem extends FileSystem {
List<FsPath> fsPaths = new ArrayList<FsPath>();
for (FileStatus f : stat) {
fsPaths.add(
- fillStorageFile(new FsPath(StorageUtils.HDFS_SCHEMA + f.getPath().toUri().getPath()), f));
+ fillStorageFile(
+ new FsPath(StorageUtils.HDFS_SCHEMA() + f.getPath().toUri().getPath()), f));
}
if (fsPaths.isEmpty()) {
return null;
@@ -174,35 +173,39 @@ public class HDFSFileSystem extends FileSystem {
@Override
public void init(Map<String, String> properties) throws IOException {
if (MapUtils.isNotEmpty(properties)
- && properties.containsKey(StorageConfiguration.PROXY_USER.key())) {
- user = StorageConfiguration.PROXY_USER.getValue(properties);
+ && properties.containsKey(StorageConfiguration.PROXY_USER().key())) {
+ user = StorageConfiguration.PROXY_USER().getValue(properties);
+ properties.remove(StorageConfiguration.PROXY_USER().key());
}
if (user == null) {
throw new IOException("User cannot be empty(用户不能为空)");
}
-
- if (label == null && (boolean) Configuration.IS_MULTIPLE_YARN_CLUSTER()) {
- label = StorageConfiguration.LINKIS_STORAGE_FS_LABEL.getValue();
+ if (label == null && Configuration.IS_MULTIPLE_YARN_CLUSTER()) {
+ label = StorageConfiguration.LINKIS_STORAGE_FS_LABEL().getValue();
}
- conf = HDFSUtils.getConfigurationByLabel(user, label);
-
+ /** if properties is null do not to create conf */
if (MapUtils.isNotEmpty(properties)) {
- for (String key : properties.keySet()) {
- String v = properties.get(key);
- if (StringUtils.isNotEmpty(v)) {
- conf.set(key, v);
+ conf = HDFSUtils.getConfigurationByLabel(user, label);
+ if (MapUtils.isNotEmpty(properties)) {
+ for (String key : properties.keySet()) {
+ String v = properties.get(key);
+ if (StringUtils.isNotEmpty(v)) {
+ conf.set(key, v);
+ }
}
}
}
- if (StorageConfiguration.FS_CACHE_DISABLE.getValue()) {
- conf.set("fs.hdfs.impl.disable.cache", "true");
+ if (null != conf) {
+ fs = HDFSUtils.getHDFSUserFileSystem(user, label, conf);
+ } else {
+ fs = HDFSUtils.getHDFSUserFileSystem(user, label);
}
- fs = HDFSUtils.getHDFSUserFileSystem(user, label, conf);
+
if (fs == null) {
throw new IOException("init HDFS FileSystem failed!");
}
- if (StorageConfiguration.FS_CHECKSUM_DISBALE.getValue()) {
+ if (StorageConfiguration.FS_CHECKSUM_DISBALE().getValue()) {
fs.setVerifyChecksum(false);
fs.setWriteChecksum(false);
}
@@ -215,7 +218,7 @@ public class HDFSFileSystem extends FileSystem {
@Override
public String rootUserName() {
- return StorageConfiguration.HDFS_ROOT_USER.getValue();
+ return StorageConfiguration.HDFS_ROOT_USER().getValue();
}
@Override
@@ -248,7 +251,6 @@ public class HDFSFileSystem extends FileSystem {
return fs.append(new Path(path));
} else {
OutputStream out = fs.create(new Path(path), true);
- this.setPermission(dest, this.getDefaultFilePerm());
return out;
}
}
@@ -293,17 +295,16 @@ public class HDFSFileSystem extends FileSystem {
@Override
public boolean canRead(FsPath dest) throws IOException {
- return canAccess(dest, FsAction.READ);
+ return canAccess(dest, FsAction.READ, this.user);
}
- @Override
- public boolean canRead(FsPath fsPath, String s) throws IOException {
- return false;
+ public boolean canRead(FsPath dest, String user) throws IOException {
+ return canAccess(dest, FsAction.READ, user);
}
@Override
public boolean canWrite(FsPath dest) throws IOException {
- return canAccess(dest, FsAction.WRITE);
+ return canAccess(dest, FsAction.WRITE, this.user);
}
@Override
@@ -313,10 +314,10 @@ public class HDFSFileSystem extends FileSystem {
} catch (IOException e) {
String message = e.getMessage();
String rootCauseMessage = ExceptionUtils.getRootCauseMessage(e);
- if ((message != null && message.matches(LinkisStorageConf.HDFS_FILE_SYSTEM_REST_ERRS))
+ if ((message != null && message.matches(LinkisStorageConf.HDFS_FILE_SYSTEM_REST_ERRS()))
|| (rootCauseMessage != null
- && rootCauseMessage.matches(LinkisStorageConf.HDFS_FILE_SYSTEM_REST_ERRS))) {
- logger.info("Failed to execute exists, retry", e);
+ && rootCauseMessage.matches(LinkisStorageConf.HDFS_FILE_SYSTEM_REST_ERRS()))) {
+ logger.info("Failed to execute exists for user {}, retry", user, e);
resetRootHdfs();
return fs.exists(new Path(checkHDFSPath(dest.getPath())));
} else {
@@ -334,8 +335,12 @@ public class HDFSFileSystem extends FileSystem {
} else {
HDFSUtils.closeHDFSFIleSystem(fs, user, label);
}
- logger.warn(user + "FS reset close.");
- fs = HDFSUtils.getHDFSUserFileSystem(user, label, conf);
+ logger.warn("{} FS reset close.", user);
+ if (null != conf) {
+ fs = HDFSUtils.getHDFSUserFileSystem(user, label, conf);
+ } else {
+ fs = HDFSUtils.getHDFSUserFileSystem(user, label);
+ }
}
}
}
@@ -362,7 +367,7 @@ public class HDFSFileSystem extends FileSystem {
@Override
public void close() throws IOException {
if (null != fs) {
- HDFSUtils.closeHDFSFIleSystem(fs, user, label);
+ HDFSUtils.closeHDFSFIleSystem(fs, user);
} else {
logger.warn("FS was null, cannot close.");
}
@@ -388,7 +393,7 @@ public class HDFSFileSystem extends FileSystem {
return fsPath;
}
- private boolean canAccess(FsPath fsPath, FsAction access) throws IOException {
+ private boolean canAccess(FsPath fsPath, FsAction access, String user) throws IOException {
String path = checkHDFSPath(fsPath.getPath());
if (!exists(fsPath)) {
throw new IOException("directory or file not exists: " + path);
@@ -396,12 +401,12 @@ public class HDFSFileSystem extends FileSystem {
FileStatus f = fs.getFileStatus(new Path(path));
FsPermission permission = f.getPermission();
- UserGroupInformation ugi = HDFSUtils.getUserGroupInformation(user, label);
+ UserGroupInformation ugi = HDFSUtils.getUserGroupInformation(user);
String[] groupNames;
try {
groupNames = ugi.getGroupNames();
} catch (NullPointerException e) {
- if ((Boolean) Configuration.IS_TEST_MODE().getValue()) {
+ if ((Boolean) org.apache.linkis.common.conf.Configuration.IS_TEST_MODE().getValue()) {
groupNames = new String[] {"hadoop"};
} else {
throw e;
@@ -433,9 +438,9 @@ public class HDFSFileSystem extends FileSystem {
private String checkHDFSPath(String path) {
try {
- boolean checkHdfsPath = (boolean) StorageConfiguration.HDFS_PATH_PREFIX_CHECK_ON.getValue();
+ boolean checkHdfsPath = (boolean) StorageConfiguration.HDFS_PATH_PREFIX_CHECK_ON().getValue();
if (checkHdfsPath) {
- boolean rmHdfsPrefix = (boolean) StorageConfiguration.HDFS_PATH_PREFIX_REMOVE.getValue();
+ boolean rmHdfsPrefix = (boolean) StorageConfiguration.HDFS_PATH_PREFIX_REMOVE().getValue();
if (rmHdfsPrefix) {
if (StringUtils.isBlank(path)) {
return path;
@@ -471,4 +476,21 @@ public class HDFSFileSystem extends FileSystem {
}
return path;
}
+
+ @Override
+ public long getLength(FsPath dest) throws IOException {
+ FileStatus fileStatus = fs.getFileStatus(new Path(checkHDFSPath(dest.getPath())));
+ return fileStatus.getLen();
+ }
+
+ @Override
+ public String checkSum(FsPath dest) throws IOException {
+ String path = checkHDFSPath(dest.getPath());
+ if (!exists(dest)) {
+ throw new IOException("directory or file not exists: " + path);
+ }
+ MD5MD5CRC32FileChecksum fileChecksum =
+ (MD5MD5CRC32FileChecksum) fs.getFileChecksum(new Path(path));
+ return fileChecksum.toString().split(":")[1];
+ }
}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java
index 9516e90361..a03a25950e 100644
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java
@@ -47,12 +47,7 @@ import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.nio.file.attribute.UserPrincipal;
import java.nio.file.attribute.UserPrincipalLookupService;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Stack;
+import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -113,7 +108,7 @@ public class LocalFileSystem extends FileSystem {
if (group != null) {
setGroup(dest, group);
}
- setGroup(dest, StorageConfiguration.STORAGE_USER_GROUP.getValue());
+ setGroup(dest, StorageConfiguration.STORAGE_USER_GROUP().getValue());
return true;
}
@@ -136,7 +131,7 @@ public class LocalFileSystem extends FileSystem {
@Override
public boolean setGroup(FsPath dest, String group) throws IOException {
- LOG.info("Set group with path:" + dest.getPath() + "and group:" + user);
+ LOG.info("Set group with path:" + dest.getPath() + " and group:" + group);
if (!StorageUtils.isIOProxy()) {
LOG.info("io not proxy, setGroup skip");
return true;
@@ -262,9 +257,17 @@ public class LocalFileSystem extends FileSystem {
LOG.info("Try to list path:" + path.getPath() + " with error msg");
if (files != null) {
List<FsPath> rtn = new ArrayList();
+ Set<String> fileNameSet = new HashSet<>();
+ fileNameSet.add(path.getPath().trim());
String message = "";
for (File f : files) {
try {
+ if (fileNameSet.contains(f.getPath())) {
+ LOG.info("File {} is duplicate", f.getPath());
+ continue;
+ } else {
+ fileNameSet.add(f.getParent().trim());
+ }
rtn.add(get(f.getPath()));
} catch (Throwable e) {
LOG.warn("Failed to list path:", e);
@@ -289,15 +292,15 @@ public class LocalFileSystem extends FileSystem {
if (MapUtils.isNotEmpty(properties)) {
this.properties = properties;
- if (properties.containsKey(StorageConfiguration.PROXY_USER.key())) {
- user = StorageConfiguration.PROXY_USER.getValue(properties);
+ if (properties.containsKey(StorageConfiguration.PROXY_USER().key())) {
+ user = StorageConfiguration.PROXY_USER().getValue(properties);
}
- group = StorageConfiguration.STORAGE_USER_GROUP.getValue(properties);
+ group = StorageConfiguration.STORAGE_USER_GROUP().getValue(properties);
} else {
this.properties = new HashMap<String, String>();
}
if (FsPath.WINDOWS) {
- group = StorageConfiguration.STORAGE_USER_GROUP.getValue(properties);
+ group = StorageConfiguration.STORAGE_USER_GROUP().getValue(properties);
}
if (StringUtils.isEmpty(group)) {
String groupInfo;
@@ -320,7 +323,7 @@ public class LocalFileSystem extends FileSystem {
@Override
public String rootUserName() {
- return StorageConfiguration.LOCAL_ROOT_USER.getValue();
+ return StorageConfiguration.LOCAL_ROOT_USER().getValue();
}
@Override
@@ -423,7 +426,7 @@ public class LocalFileSystem extends FileSystem {
}
@Override
- public boolean canRead(FsPath fsPath, String s) throws IOException {
+ public boolean canRead(FsPath dest, String user) throws IOException {
return false;
}
@@ -494,4 +497,16 @@ public class LocalFileSystem extends FileSystem {
PosixFileAttributes attr = Files.readAttributes(Paths.get(path), PosixFileAttributes.class);
return attr.owner().getName();
}
+
+ @Override
+ public long getLength(FsPath dest) throws IOException {
+ String path = dest.getPath();
+ LOG.info("Get file length with path:" + path);
+ return new File(path).length();
+ }
+
+ @Override
+ public String checkSum(FsPath dest) {
+ return null;
+ }
}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/OSSFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/OSSFileSystem.java
deleted file mode 100644
index 2d52b83049..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/OSSFileSystem.java
+++ /dev/null
@@ -1,398 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.fs.impl;
-
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.hadoop.common.utils.HDFSUtils;
-import org.apache.linkis.storage.conf.LinkisStorageConf;
-import org.apache.linkis.storage.domain.FsPathListWithError;
-import org.apache.linkis.storage.fs.FileSystem;
-import org.apache.linkis.storage.utils.StorageConfiguration;
-import org.apache.linkis.storage.utils.StorageUtils;
-
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import com.google.common.collect.Maps;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class OSSFileSystem extends FileSystem {
-
- public static final String OSS_PREFIX = "oss://";
- private org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem fs = null;
- private Configuration conf = null;
-
- private String label = null;
-
- private static final Logger logger = LoggerFactory.getLogger(OSSFileSystem.class);
-
- /** File System abstract method start */
- @Override
- public String listRoot() throws IOException {
- return "/";
- }
-
- @Override
- public long getTotalSpace(FsPath dest) throws IOException {
- return 0;
- }
-
- @Override
- public long getFreeSpace(FsPath dest) throws IOException {
- return 0;
- }
-
- @Override
- public long getUsableSpace(FsPath dest) throws IOException {
- return 0;
- }
-
- @Override
- public boolean canExecute(FsPath dest) throws IOException {
- return true;
- }
-
- @Override
- public boolean setOwner(FsPath dest, String user, String group) throws IOException {
- return true;
- }
-
- @Override
- public boolean setOwner(FsPath dest, String user) throws IOException {
- return true;
- }
-
- @Override
- public boolean setGroup(FsPath dest, String group) throws IOException {
- return true;
- }
-
- @Override
- public boolean mkdir(FsPath dest) throws IOException {
- String path = checkOSSPath(dest.getPath());
- if (!canExecute(getParentPath(path))) {
- throw new IOException("You have not permission to access path " + path);
- }
- boolean result =
- fs.mkdirs(new Path(path), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
- this.setPermission(new FsPath(path), this.getDefaultFolderPerm());
- return result;
- }
-
- @Override
- public boolean mkdirs(FsPath dest) throws IOException {
- String path = checkOSSPath(dest.getPath());
- FsPath parentPath = getParentPath(path);
- while (!exists(parentPath)) {
- parentPath = getParentPath(parentPath.getPath());
- }
- return fs.mkdirs(new Path(path), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
- }
-
- @Override
- public boolean setPermission(FsPath dest, String permission) throws IOException {
- return true;
- }
-
- @Override
- public FsPathListWithError listPathWithError(FsPath path) throws IOException {
- FileStatus[] stat = fs.listStatus(new Path(checkOSSPath(path.getPath())));
- List<FsPath> fsPaths = new ArrayList<FsPath>();
- for (FileStatus f : stat) {
- fsPaths.add(
- fillStorageFile(
- new FsPath(
- StorageUtils.OSS_SCHEMA
- + StorageConfiguration.OSS_ACCESS_BUCKET_NAME.getValue()
- + "/"
- + f.getPath().toUri().getPath()),
- f));
- }
- if (fsPaths.isEmpty()) {
- return null;
- }
- return new FsPathListWithError(fsPaths, "");
- }
-
- /** FS interface method start */
- @Override
- public void init(Map<String, String> properties) throws IOException {
- // read origin configs from hadoop conf
- if (label == null
- && (boolean) org.apache.linkis.common.conf.Configuration.IS_MULTIPLE_YARN_CLUSTER()) {
- label = StorageConfiguration.LINKIS_STORAGE_FS_LABEL.getValue();
- }
- conf = HDFSUtils.getConfigurationByLabel(user, label);
-
- // origin configs
- Map<String, String> originProperties = Maps.newHashMap();
- originProperties.put("fs.oss.endpoint", StorageConfiguration.OSS_ENDPOINT.getValue());
- originProperties.put("fs.oss.accessKeyId", StorageConfiguration.OSS_ACCESS_KEY_ID.getValue());
- originProperties.put(
- "fs.oss.accessKeySecret", StorageConfiguration.OSS_ACCESS_KEY_SECRET.getValue());
- for (String key : originProperties.keySet()) {
- String value = originProperties.get(key);
- if (StringUtils.isNotBlank(value)) {
- conf.set(key, value);
- }
- }
-
- // additional configs
- if (MapUtils.isNotEmpty(properties)) {
- for (String key : properties.keySet()) {
- String v = properties.get(key);
- if (StringUtils.isNotBlank(v)) {
- conf.set(key, v);
- }
- }
- }
- fs = new AliyunOSSFileSystem();
- try {
- fs.initialize(
- new URI(StorageUtils.OSS_SCHEMA + StorageConfiguration.OSS_ACCESS_BUCKET_NAME.getValue()),
- conf);
- } catch (URISyntaxException e) {
- throw new IOException("init OSS FileSystem failed!");
- }
- if (fs == null) {
- throw new IOException("init OSS FileSystem failed!");
- }
- }
-
- @Override
- public String fsName() {
- return StorageUtils.OSS;
- }
-
- @Override
- public String rootUserName() {
- return null;
- }
-
- @Override
- public FsPath get(String dest) throws IOException {
- String realPath = checkOSSPath(dest);
- return fillStorageFile(new FsPath(realPath), fs.getFileStatus(new Path(realPath)));
- }
-
- @Override
- public InputStream read(FsPath dest) throws IOException {
- if (!canRead(dest)) {
- throw new IOException("You have not permission to access path " + dest.getPath());
- }
- return fs.open(new Path(dest.getPath()), 128);
- }
-
- @Override
- public OutputStream write(FsPath dest, boolean overwrite) throws IOException {
- String path = checkOSSPath(dest.getPath());
- if (!exists(dest)) {
- if (!canWrite(dest.getParent())) {
- throw new IOException("You have not permission to access path " + dest.getParent());
- }
- } else {
- if (!canWrite(dest)) {
- throw new IOException("You have not permission to access path " + path);
- }
- }
- OutputStream out =
- fs.create(
- new Path(path),
- new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL),
- overwrite,
- 0,
- (short) 0,
- 0L,
- null);
- this.setPermission(dest, this.getDefaultFilePerm());
- return out;
- }
-
- @Override
- public boolean create(String dest) throws IOException {
- if (!canExecute(getParentPath(dest))) {
- throw new IOException("You have not permission to access path " + dest);
- }
- // to do
- boolean result = fs.createNewFile(new Path(checkOSSPath(dest)));
- this.setPermission(new FsPath(dest), this.getDefaultFilePerm());
- return result;
- }
-
- @Override
- public boolean copy(String origin, String dest) throws IOException {
- if (!canExecute(getParentPath(dest))) {
- throw new IOException("You have not permission to access path " + dest);
- }
- boolean res =
- FileUtil.copy(
- fs,
- new Path(checkOSSPath(origin)),
- fs,
- new Path(checkOSSPath(dest)),
- false,
- true,
- fs.getConf());
- this.setPermission(new FsPath(dest), this.getDefaultFilePerm());
- return res;
- }
-
- @Override
- public List<FsPath> list(FsPath path) throws IOException {
- FileStatus[] stat = fs.listStatus(new Path(checkOSSPath(path.getPath())));
- List<FsPath> fsPaths = new ArrayList<FsPath>();
- for (FileStatus f : stat) {
- fsPaths.add(fillStorageFile(new FsPath(f.getPath().toUri().toString()), f));
- }
- return fsPaths;
- }
-
- @Override
- public boolean canRead(FsPath dest) throws IOException {
- return true;
- }
-
- @Override
- public boolean canRead(FsPath fsPath, String s) throws IOException {
- return false;
- }
-
- @Override
- public boolean canWrite(FsPath dest) throws IOException {
- return true;
- }
-
- @Override
- public boolean exists(FsPath dest) throws IOException {
- try {
- return fs.exists(new Path(checkOSSPath(dest.getPath())));
- } catch (IOException e) {
- String message = e.getMessage();
- String rootCauseMessage = ExceptionUtils.getRootCauseMessage(e);
- if ((message != null && message.matches(LinkisStorageConf.HDFS_FILE_SYSTEM_REST_ERRS))
- || (rootCauseMessage != null
- && rootCauseMessage.matches(LinkisStorageConf.HDFS_FILE_SYSTEM_REST_ERRS))) {
- logger.info("Failed to execute exists, retry", e);
- resetRootOSS();
- return fs.exists(new Path(checkOSSPath(dest.getPath())));
- } else {
- throw e;
- }
- }
- }
-
- private void resetRootOSS() throws IOException {
- if (fs != null) {
- synchronized (this) {
- if (fs != null) {
- fs.close();
- logger.warn(user + " FS reset close.");
- init(null);
- }
- }
- }
- }
-
- @Override
- public boolean delete(FsPath dest) throws IOException {
- String path = checkOSSPath(dest.getPath());
- return fs.delete(new Path(path), true);
- }
-
- @Override
- public boolean renameTo(FsPath oldDest, FsPath newDest) throws IOException {
- return fs.rename(
- new Path(checkOSSPath(oldDest.getPath())), new Path(checkOSSPath(newDest.getPath())));
- }
-
- @Override
- public void close() throws IOException {
- if (null != fs) {
- fs.close();
- } else {
- logger.warn("FS was null, cannot close.");
- }
- }
-
- /** Utils method start */
- private FsPath fillStorageFile(FsPath fsPath, FileStatus fileStatus) throws IOException {
- fsPath.setAccess_time(fileStatus.getAccessTime());
- fsPath.setModification_time(fileStatus.getModificationTime());
- fsPath.setOwner(fileStatus.getOwner());
- fsPath.setGroup(fileStatus.getGroup());
- fsPath.setIsdir(fileStatus.isDirectory());
- return fsPath;
- }
-
- public String getLabel() {
- return label;
- }
-
- public void setLabel(String label) {
- this.label = label;
- }
-
- private static String checkOSSPath(String path) {
- try {
- boolean checkOSSPath = (boolean) StorageConfiguration.OSS_PATH_PREFIX_CHECK_ON.getValue();
- if (checkOSSPath) {
- boolean rmOSSPrefix = (boolean) StorageConfiguration.OSS_PATH_PREFIX_REMOVE.getValue();
- if (rmOSSPrefix) {
- if (StringUtils.isBlank(path)) {
- return path;
- }
- if (path.startsWith(OSS_PREFIX)) {
- int remainIndex = OSS_PREFIX.length();
- String[] t1 = path.substring(remainIndex).split("/", 2);
- if (t1.length != 2) {
- logger.warn("checkOSSPath Invalid path: " + path);
- return path;
- }
- if (logger.isDebugEnabled()) {
- logger.debug("checkOSSPath ori path : {}, after path : {}", path, "/" + t1[1]);
- }
- return "/" + t1[1];
- } else {
- return path;
- }
- }
- }
- } catch (Exception e) {
- logger.warn("checkOSSPath error. msg : " + e.getMessage() + " ", e);
- }
- return path;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/S3FileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/S3FileSystem.java
deleted file mode 100644
index 1e7a90532c..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/S3FileSystem.java
+++ /dev/null
@@ -1,464 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.fs.impl;
-
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.storage.domain.FsPathListWithError;
-import org.apache.linkis.storage.exception.StorageWarnException;
-import org.apache.linkis.storage.fs.FileSystem;
-import org.apache.linkis.storage.utils.StorageConfiguration;
-import org.apache.linkis.storage.utils.StorageUtils;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.*;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.client.builder.AwsClientBuilder;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.amazonaws.services.s3.model.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.TO_BE_UNKNOW;
-
-public class S3FileSystem extends FileSystem {
- private static final Logger logger = LoggerFactory.getLogger(S3FileSystem.class);
- private String accessKey;
- private String secretKey;
-
- private String endPoint;
-
- private String region;
-
- private String bucket;
-
- private String label;
-
- private AmazonS3 s3Client;
-
- private static final String INIT_FILE_NAME = ".s3_dir_init";
-
- @Override
- public void init(Map<String, String> properties) throws IOException {
- accessKey = StorageConfiguration.S3_ACCESS_KEY.getValue(properties);
- secretKey = StorageConfiguration.S3_SECRET_KEY.getValue(properties);
- endPoint = StorageConfiguration.S3_ENDPOINT.getValue(properties);
- bucket = StorageConfiguration.S3_BUCKET.getValue(properties);
- region = StorageConfiguration.S3_REGION.getValue(properties);
-
- AwsClientBuilder.EndpointConfiguration endpointConfiguration =
- new AwsClientBuilder.EndpointConfiguration(endPoint, region);
-
- BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(accessKey, secretKey);
-
- AWSStaticCredentialsProvider StaticCredentials =
- new AWSStaticCredentialsProvider(basicAWSCredentials);
-
- s3Client =
- AmazonS3ClientBuilder.standard()
- .withEndpointConfiguration(endpointConfiguration)
- .withPathStyleAccessEnabled(true)
- .withCredentials(StaticCredentials)
- .build();
- }
-
- @Override
- public String fsName() {
- return StorageUtils.S3;
- }
-
- @Override
- public String rootUserName() {
- return null;
- }
-
- @Override
- public FsPath get(String dest) throws IOException {
- FsPath ret = new FsPath(dest);
- if (exists(ret)) {
- return ret;
- } else {
- logger.warn("File or folder does not exist or file name is garbled(文件或者文件夹不存在或者文件名乱码)");
- throw new StorageWarnException(
- TO_BE_UNKNOW.getErrorCode(),
- "File or folder does not exist or file name is garbled(文件或者文件夹不存在或者文件名乱码)");
- }
- }
-
- @Override
- public InputStream read(FsPath dest) throws IOException {
- try {
- return s3Client.getObject(bucket, buildPrefix(dest.getPath(), false)).getObjectContent();
- } catch (AmazonS3Exception e) {
- throw new IOException("You have not permission to access path " + dest.getPath());
- }
- }
-
- @Override
- public OutputStream write(FsPath dest, boolean overwrite) throws IOException {
- try (InputStream inputStream = read(dest);
- OutputStream outputStream =
- new S3OutputStream(s3Client, bucket, buildPrefix(dest.getPath(), false))) {
- if (!overwrite) {
- IOUtils.copy(inputStream, outputStream);
- }
- return outputStream;
- }
- }
-
- @Override
- public boolean create(String dest) throws IOException {
- if (exists(new FsPath(dest))) {
- return false;
- }
- s3Client.putObject(bucket, dest, "");
- return true;
- }
-
- @Override
- public List<FsPath> list(FsPath path) throws IOException {
- try {
- if (!StringUtils.isEmpty(path.getPath())) {
- ListObjectsV2Result listObjectsV2Result = s3Client.listObjectsV2(bucket, path.getPath());
- List<S3ObjectSummary> s3ObjectSummaries = listObjectsV2Result.getObjectSummaries();
- return s3ObjectSummaries.stream()
- .filter(summary -> !isInitFile(summary))
- .map(
- summary -> {
- FsPath newPath = new FsPath(buildPath(summary.getKey()));
- return fillStorageFile(newPath, summary);
- })
- .collect(Collectors.toList());
- }
- } catch (AmazonS3Exception e) {
- throw new IOException("You have not permission to access path " + path.getPath());
- }
-
- return new ArrayList<>();
- }
-
- @Override
- public FsPathListWithError listPathWithError(FsPath path) throws IOException {
- return listPathWithError(path, true);
- }
-
- public FsPathListWithError listPathWithError(FsPath path, boolean ignoreInitFile)
- throws IOException {
- List<FsPath> rtn = new ArrayList<>();
- try {
- if (!StringUtils.isEmpty(path.getPath())) {
- ListObjectsV2Request listObjectsV2Request =
- new ListObjectsV2Request()
- .withBucketName(bucket)
- .withPrefix(buildPrefix(path.getPath()))
- .withDelimiter("/");
- ListObjectsV2Result dirResult = s3Client.listObjectsV2(listObjectsV2Request);
- List<S3ObjectSummary> s3ObjectSummaries = dirResult.getObjectSummaries();
- List<String> commonPrefixes = dirResult.getCommonPrefixes();
- if (s3ObjectSummaries != null) {
- for (S3ObjectSummary summary : s3ObjectSummaries) {
- if (isInitFile(summary) && ignoreInitFile) continue;
- FsPath newPath = new FsPath(buildPath(summary.getKey()));
- rtn.add(fillStorageFile(newPath, summary));
- }
- }
- if (commonPrefixes != null) {
- for (String dir : commonPrefixes) {
- FsPath newPath = new FsPath(buildPath(dir));
- newPath.setIsdir(true);
- rtn.add(newPath);
- }
- }
- return new FsPathListWithError(rtn, "");
- }
- } catch (AmazonS3Exception e) {
- throw new IOException("You have not permission to access path " + path.getPath());
- }
-
- return null;
- }
-
- @Override
- public boolean exists(FsPath dest) throws IOException {
- try {
- if (new File(dest.getPath()).getName().contains(".")) {
- return existsFile(dest);
- }
- ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request();
- listObjectsV2Request
- .withBucketName(bucket)
- .withPrefix(buildPrefix(dest.getPath()))
- .withDelimiter("/");
- return s3Client.listObjectsV2(listObjectsV2Request).getObjectSummaries().size()
- + s3Client.listObjectsV2(listObjectsV2Request).getCommonPrefixes().size()
- > 0;
- } catch (AmazonS3Exception e) {
- return false;
- }
- }
-
- public boolean existsFile(FsPath dest) {
- try {
- return s3Client.doesObjectExist(bucket, buildPrefix(dest.getPath(), false));
- } catch (AmazonS3Exception e) {
- return false;
- }
- }
-
- @Override
- public boolean delete(FsPath dest) throws IOException {
- try {
- ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request();
- listObjectsV2Request.withBucketName(bucket).withPrefix(buildPrefix(dest.getPath(), false));
- ListObjectsV2Result result = s3Client.listObjectsV2(listObjectsV2Request);
- String[] keyList =
- result.getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new);
- DeleteObjectsRequest deleteObjectsRequest =
- new DeleteObjectsRequest("test").withKeys(keyList);
- s3Client.deleteObjects(deleteObjectsRequest);
- return true;
- } catch (AmazonS3Exception e) {
- throw new IOException("You have not permission to access path " + dest.getPath());
- }
- }
-
- @Override
- public boolean renameTo(FsPath oldDest, FsPath newDest) throws IOException {
- try {
- String newOriginPath = buildPrefix(oldDest.getPath(), false);
- String newDestPath = buildPrefix(newDest.getPath(), false);
- ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request();
- listObjectsV2Request.withBucketName(bucket).withPrefix(newOriginPath);
- ListObjectsV2Result result = s3Client.listObjectsV2(listObjectsV2Request);
- List<String> keyList =
- result.getObjectSummaries().stream()
- .map(S3ObjectSummary::getKey)
- .collect(Collectors.toList());
- List<String> newKeyList =
- keyList.stream()
- .map(key -> key.replaceFirst(newOriginPath, newDestPath))
- .collect(Collectors.toList());
- for (int i = 0; i < keyList.size(); i++) {
- String key = keyList.get(i);
- String newKey = newKeyList.get(i);
- s3Client.copyObject(bucket, key, bucket, newKey);
- s3Client.deleteObject(bucket, key);
- }
- return true;
- } catch (AmazonS3Exception e) {
- s3Client.deleteObject(bucket, newDest.getPath());
- throw new IOException(
- "You have not permission to access path "
- + oldDest.getPath()
- + " or "
- + newDest.getPath());
- }
- }
-
- @Override
- public boolean copy(String origin, String dest) throws IOException {
- try {
- String newOrigin = buildPrefix(origin, false);
- String newDest = buildPrefix(dest, false);
- ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request();
- listObjectsV2Request.withBucketName(bucket).withPrefix(newOrigin);
- ListObjectsV2Result result = s3Client.listObjectsV2(listObjectsV2Request);
- List<String> keyList =
- result.getObjectSummaries().stream()
- .map(S3ObjectSummary::getKey)
- .collect(Collectors.toList());
- List<String> newKeyList =
- keyList.stream()
- .map(key -> key.replaceFirst(newOrigin, newDest))
- .collect(Collectors.toList());
- for (int i = 0; i < keyList.size(); i++) {
- String key = keyList.get(i);
- String newKey = newKeyList.get(i);
- s3Client.copyObject(bucket, key, bucket, newKey);
- }
- return true;
- } catch (AmazonS3Exception e) {
- throw new IOException("You have not permission to access path " + origin + " or " + dest);
- }
- }
-
- private boolean isDir(S3ObjectSummary s3ObjectSummary, String prefix) {
- return s3ObjectSummary.getKey().substring(prefix.length()).contains("/");
- }
-
- private boolean isInitFile(S3ObjectSummary s3ObjectSummary) {
- return s3ObjectSummary.getKey().contains(INIT_FILE_NAME);
- }
-
- @Override
- public String listRoot() {
- return "/";
- }
-
- @Override
- public boolean mkdir(FsPath dest) throws IOException {
- String path = new File(dest.getPath(), INIT_FILE_NAME).getPath();
- if (exists(new FsPath(path))) {
- return false;
- }
- return create(path);
- }
-
- @Override
- public boolean mkdirs(FsPath dest) throws IOException {
- return mkdir(dest);
- }
-
- private FsPath fillStorageFile(FsPath fsPath, S3ObjectSummary s3ObjectSummary) {
- fsPath.setModification_time(s3ObjectSummary.getLastModified().getTime());
- Owner owner = s3ObjectSummary.getOwner();
- if (owner != null) {
- fsPath.setOwner(owner.getDisplayName());
- }
- try {
- fsPath.setIsdir(isDir(s3ObjectSummary, fsPath.getParent().getPath()));
- } catch (Throwable e) {
- logger.warn("Failed to fill storage file:" + fsPath.getPath(), e);
- }
-
- if (fsPath.isdir()) {
- fsPath.setLength(0);
- } else {
- fsPath.setLength(s3ObjectSummary.getSize());
- }
- return fsPath;
- }
-
- @Override
- public boolean canRead(FsPath dest) {
- return true;
- }
-
- @Override
- public boolean canRead(FsPath fsPath, String s) throws IOException {
- return false;
- }
-
- @Override
- public boolean canWrite(FsPath dest) {
- return true;
- }
-
- @Override
- public long getTotalSpace(FsPath dest) {
- return 0;
- }
-
- @Override
- public long getFreeSpace(FsPath dest) {
- return 0;
- }
-
- @Override
- public long getUsableSpace(FsPath dest) {
- return 0;
- }
-
- @Override
- public boolean canExecute(FsPath dest) {
- return true;
- }
-
- @Override
- public boolean setOwner(FsPath dest, String user, String group) {
- return false;
- }
-
- @Override
- public boolean setOwner(FsPath dest, String user) {
- return false;
- }
-
- @Override
- public boolean setGroup(FsPath dest, String group) {
- return false;
- }
-
- @Override
- public boolean setPermission(FsPath dest, String permission) {
- return false;
- }
-
- @Override
- public void close() throws IOException {}
-
- public String getLabel() {
- return label;
- }
-
- public void setLabel(String label) {
- this.label = label;
- }
-
- public String buildPath(String path) {
- if (path == null || "".equals(path)) return "";
- if (path.startsWith("/")) {
- return StorageUtils.S3_SCHEMA + path;
- }
- return StorageUtils.S3_SCHEMA + "/" + path;
- }
-
- public String buildPrefix(String path, boolean addTail) {
- String res = path;
- if (path == null || "".equals(path)) return "";
- if (path.startsWith("/")) {
- res = path.replaceFirst("/", "");
- }
- if (!path.endsWith("/") && addTail) {
- res = res + "/";
- }
- return res;
- }
-
- public String buildPrefix(String path) {
- return buildPrefix(path, true);
- }
-}
-
-class S3OutputStream extends ByteArrayOutputStream {
- private AmazonS3 s3Client;
- private String bucket;
- private String path;
-
- public S3OutputStream(AmazonS3 s3Client, String bucket, String path) {
- this.s3Client = s3Client;
- this.bucket = bucket;
- this.path = path;
- }
-
- @Override
- public void close() throws IOException {
- byte[] buffer = this.toByteArray();
- try (InputStream in = new ByteArrayInputStream(buffer)) {
- s3Client.putObject(bucket, path, in, new ObjectMetadata());
- }
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOClient.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOClient.java
deleted file mode 100644
index 294f9957c9..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOClient.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.io;
-
-import org.apache.linkis.storage.domain.MethodEntity;
-
-import java.util.Map;
-
-public interface IOClient {
-
- String execute(String user, MethodEntity methodEntity, Map<String, Object> params);
-
- String[] executeWithEngine(String user, MethodEntity methodEntity, Map<String, Object> params);
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOClientFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOClientFactory.java
deleted file mode 100644
index 244ad59477..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOClientFactory.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.io;
-
-import org.apache.linkis.storage.exception.StorageErrorException;
-
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.MUST_REGISTER_TOC;
-
-public class IOClientFactory {
- private static final Logger logger = LoggerFactory.getLogger(IOClientFactory.class);
- private static IOClient ioClient = null;
-
- private static final String SUCCESS = "SUCCESS";
- private static final String FAILED = "FAILED";
-
- public static IOClient getIOClient() throws StorageErrorException {
- if (ioClient == null) {
- throw new StorageErrorException(
- MUST_REGISTER_TOC.getErrorCode(), MUST_REGISTER_TOC.getErrorDesc());
- }
- return ioClient;
- }
-
- /**
- * This method is called when ioClient is initialized. ioClient初始化时会调用该方法
- *
- * @param client IOClient
- */
- public static void register(IOClient client) {
- ioClient = client;
- logger.debug("IOClient: {} registered", ioClient.toString());
- }
-
- public static String getFSId() {
- return UUID.randomUUID().toString();
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOMethodInterceptorCreator.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOMethodInterceptorCreator.java
deleted file mode 100644
index 26a8ceea10..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOMethodInterceptorCreator.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.io;
-
-import org.springframework.cglib.proxy.MethodInterceptor;
-
-public interface IOMethodInterceptorCreator {
-
- MethodInterceptor createIOMethodInterceptor(String fsName);
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOMethodInterceptorFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOMethodInterceptorFactory.java
deleted file mode 100644
index 8f0c4016d1..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOMethodInterceptorFactory.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.io;
-
-import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary;
-import org.apache.linkis.storage.exception.StorageWarnException;
-
-import org.springframework.cglib.proxy.MethodInterceptor;
-
-public class IOMethodInterceptorFactory {
-
- private static IOMethodInterceptorCreator interceptorCreator = null;
-
- private IOMethodInterceptorFactory() {}
-
- /**
- * This method is called when ioClient is initialized. ioClient初始化时会调用该方法
- *
- * @param interceptorCreator
- */
- public static void register(IOMethodInterceptorCreator interceptorCreator) {
- IOMethodInterceptorFactory.interceptorCreator = interceptorCreator;
- }
-
- public static MethodInterceptor getIOMethodInterceptor(String fsName)
- throws StorageWarnException {
- if (IOMethodInterceptorFactory.interceptorCreator == null) {
- throw new StorageWarnException(
- LinkisStorageErrorCodeSummary.MUST_REGISTER_TOM.getErrorCode(),
- LinkisStorageErrorCodeSummary.MUST_REGISTER_TOM.getErrorDesc());
- }
- return IOMethodInterceptorFactory.interceptorCreator.createIOMethodInterceptor(fsName);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/DefaultResultSetFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/DefaultResultSetFactory.java
deleted file mode 100644
index da51a2f13d..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/DefaultResultSetFactory.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset;
-
-import org.apache.linkis.common.io.Fs;
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.common.io.resultset.ResultSet;
-import org.apache.linkis.storage.FSFactory;
-import org.apache.linkis.storage.conf.LinkisStorageConf;
-import org.apache.linkis.storage.domain.Dolphin;
-import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary;
-import org.apache.linkis.storage.exception.StorageWarnException;
-import org.apache.linkis.storage.utils.StorageConfiguration;
-import org.apache.linkis.storage.utils.StorageUtils;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.text.MessageFormat;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.THE_FILE_IS_EMPTY;
-
-public class DefaultResultSetFactory implements ResultSetFactory {
-
- private static final Logger logger = LoggerFactory.getLogger(DefaultResultSetFactory.class);
-
- private final Map<String, Class<ResultSet<ResultMetaData, ResultRecord>>> resultClasses;
-
- private final String[] resultTypes;
-
- public DefaultResultSetFactory() {
- resultClasses =
- StorageUtils.loadClasses(
- StorageConfiguration.STORAGE_RESULT_SET_CLASSES.getValue(),
- StorageConfiguration.STORAGE_RESULT_SET_PACKAGE.getValue(),
- t -> {
- try {
- return t.newInstance().resultSetType().toLowerCase(Locale.getDefault());
- } catch (InstantiationException e) {
- logger.warn("DefaultResultSetFactory init failed", e);
- } catch (IllegalAccessException e) {
- logger.warn("DefaultResultSetFactory init failed", e);
- }
- return null;
- });
- resultTypes = ResultSetFactory.resultSetType.keySet().toArray(new String[0]);
- }
-
- @Override
- public ResultSet<? extends MetaData, ? extends Record> getResultSetByType(String resultSetType) {
- if (!resultClasses.containsKey(resultSetType)) {
- throw new StorageWarnException(
- LinkisStorageErrorCodeSummary.UNSUPPORTED_RESULT.getErrorCode(),
- MessageFormat.format(
- LinkisStorageErrorCodeSummary.UNSUPPORTED_RESULT.getErrorDesc(), resultSetType));
- }
- try {
- return resultClasses.get(resultSetType).newInstance();
- } catch (InstantiationException | IllegalAccessException e) {
- throw new StorageWarnException(
- LinkisStorageErrorCodeSummary.UNSUPPORTED_RESULT.getErrorCode(),
- MessageFormat.format(
- LinkisStorageErrorCodeSummary.UNSUPPORTED_RESULT.getErrorDesc(), resultSetType),
- e);
- }
- }
-
- @Override
- public ResultSet<? extends MetaData, ? extends Record> getResultSetByPath(FsPath fsPath)
- throws StorageWarnException {
- return getResultSetByPath(fsPath, StorageUtils.getJvmUser());
- }
-
- @Override
- public ResultSet<? extends MetaData, ? extends Record> getResultSetByContent(String content) {
- return getResultSetByType(Dolphin.getType(content));
- }
-
- @Override
- public boolean exists(String resultSetType) {
- return resultClasses.containsKey(resultSetType);
- }
-
- @Override
- public boolean isResultSetPath(String path) {
- return path.endsWith(Dolphin.DOLPHIN_FILE_SUFFIX)
- || path.endsWith(LinkisStorageConf.PARQUET_FILE_SUFFIX)
- || path.endsWith(LinkisStorageConf.ORC_FILE_SUFFIX);
- }
-
- @Override
- public boolean isResultSet(String content) {
- try {
- return resultClasses.containsKey(Dolphin.getType(content));
- } catch (Exception e) {
- logger.info("Wrong result Set: " + e.getMessage());
- return false;
- }
- }
-
- @Override
- public ResultSet<? extends MetaData, ? extends Record> getResultSet(String output)
- throws StorageWarnException {
- return getResultSet(output, StorageUtils.getJvmUser());
- }
-
- @Override
- public String[] getResultSetType() {
- return Arrays.copyOf(resultTypes, resultTypes.length);
- }
-
- @Override
- public ResultSet<? extends MetaData, ? extends Record> getResultSetByPath(FsPath fsPath, Fs fs) {
- ResultSet resultSet = null;
- try (InputStream inputStream = fs.read(fsPath)) {
- if (fsPath.getPath().endsWith(Dolphin.DOLPHIN_FILE_SUFFIX)) {
- String resultSetType = Dolphin.getType(inputStream);
- if (StringUtils.isEmpty(resultSetType)) {
- throw new StorageWarnException(
- THE_FILE_IS_EMPTY.getErrorCode(),
- MessageFormat.format(THE_FILE_IS_EMPTY.getErrorDesc(), fsPath.getPath()));
- }
- // Utils.tryQuietly(fs::close);
- resultSet = getResultSetByType(resultSetType);
- } else if (fsPath.getPath().endsWith(LinkisStorageConf.PARQUET_FILE_SUFFIX)
- || fsPath.getPath().endsWith(LinkisStorageConf.ORC_FILE_SUFFIX)) {
- resultSet = getResultSetByType(ResultSetFactory.TABLE_TYPE);
- }
- return resultSet;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public ResultSet<? extends MetaData, ? extends Record> getResultSetByPath(
- FsPath fsPath, String proxyUser) {
- if (fsPath == null) {
- return null;
- }
- logger.info("Get Result Set By Path:" + fsPath.getPath());
- try (Fs fs = FSFactory.getFsByProxyUser(fsPath, proxyUser)) {
- fs.init(new HashMap<>());
- try (InputStream inputStream = fs.read(fsPath)) {
- String resultSetType = Dolphin.getType(inputStream);
- if (StringUtils.isEmpty(resultSetType)) {
- throw new StorageWarnException(
- THE_FILE_IS_EMPTY.getErrorCode(),
- MessageFormat.format(THE_FILE_IS_EMPTY.getErrorDesc(), fsPath.getPath()));
- }
- IOUtils.closeQuietly(inputStream);
- return getResultSetByType(resultSetType);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public ResultSet<? extends MetaData, ? extends Record> getResultSet(
- String output, String proxyUser) {
- if (isResultSetPath(output)) {
- return getResultSetByPath(new FsPath(output), proxyUser);
- } else if (isResultSet(output)) {
- return getResultSetByContent(output);
- } else {
- return null;
- }
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetReader.java
deleted file mode 100644
index 249e326cde..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetReader.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset;
-
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.common.io.resultset.ResultSet;
-import org.apache.linkis.common.io.resultset.ResultSetReader;
-import org.apache.linkis.storage.domain.Column;
-import org.apache.linkis.storage.domain.DataType;
-import org.apache.linkis.storage.resultset.table.TableMetaData;
-import org.apache.linkis.storage.resultset.table.TableRecord;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.orc.OrcFile;
-import org.apache.orc.Reader;
-import org.apache.orc.RecordReader;
-import org.apache.orc.TypeDescription;
-import org.apache.orc.storage.ql.exec.vector.*;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class OrcResultSetReader<K extends MetaData, V extends Record>
- extends ResultSetReader<K, V> {
-
- private static final Logger logger = LoggerFactory.getLogger(OrcResultSetReader.class);
-
- private FsPath fsPath;
-
- private final ResultSet<K, V> resultSet;
-
- private final InputStream inputStream;
-
- private MetaData metaData;
-
- private Record row;
-
- private RecordReader rows;
-
- private Reader reader;
-
- public OrcResultSetReader(ResultSet resultSet, InputStream inputStream, FsPath fsPath)
- throws IOException {
- super(resultSet, inputStream);
- this.resultSet = resultSet;
- this.inputStream = inputStream;
- this.fsPath = fsPath;
- this.reader =
- OrcFile.createReader(
- new Path(fsPath.getPath()), OrcFile.readerOptions(new Configuration()));
- this.rows = reader.rows();
- }
-
- @Override
- public MetaData getMetaData() {
- if (metaData == null) {
- try {
- List<String> fieldNames = reader.getSchema().getFieldNames();
- List<TypeDescription> typeDescriptions = reader.getSchema().getChildren();
- List<Column> columnList = new ArrayList<>();
- for (int i = 0; i < fieldNames.size(); i++) {
- Column column =
- new Column(
- fieldNames.get(i),
- DataType.toDataType(typeDescriptions.get(i).getCategory().getName()),
- "");
- columnList.add(column);
- }
-
- metaData = new TableMetaData(columnList.toArray(new Column[0]));
- } catch (Exception e) {
- throw new RuntimeException("Failed to read parquet schema", e);
- }
- }
- return metaData;
- }
-
- @Override
- public int skip(int recordNum) throws IOException {
- if (recordNum < 0) return -1;
-
- for (int i = recordNum; i > 0; i--) {
- try {
- hasNext();
- } catch (Throwable t) {
- return recordNum - i;
- }
- }
- return recordNum;
- }
-
- @Override
- public long getPosition() throws IOException {
- throw new UnsupportedOperationException("Storeage Unsupported type: getPosition");
- }
-
- @Override
- public long available() throws IOException {
- throw new UnsupportedOperationException("Storeage Unsupported type: available");
- }
-
- @Override
- public boolean hasNext() throws IOException {
- if (metaData == null) getMetaData();
- if (rows == null) return false;
- VectorizedRowBatch batch =
- reader.getSchema().createRowBatch(TypeDescription.RowBatchVersion.ORIGINAL, 1);
- TableMetaData tableMetaData = (TableMetaData) metaData;
-
- if (rows.nextBatch(batch)) {
- int rowNum = 0;
- Object[] rowData = new Object[tableMetaData.getColumns().length];
- for (int i = 0; i < batch.numCols; i++) {
- ColumnVector columnVector = batch.cols[i];
- if (columnVector instanceof BytesColumnVector) {
- BytesColumnVector vector = (BytesColumnVector) columnVector;
- rowData[i] = vector.toString(rowNum);
- } else if (columnVector instanceof Decimal64ColumnVector) {
- Decimal64ColumnVector vector = (Decimal64ColumnVector) columnVector;
- rowData[i] = vector.vector[rowNum];
- } else if (columnVector instanceof DecimalColumnVector) {
- DecimalColumnVector vector = (DecimalColumnVector) columnVector;
- rowData[i] = vector.vector[rowNum];
- } else if (columnVector instanceof DoubleColumnVector) {
- DoubleColumnVector vector = (DoubleColumnVector) columnVector;
- rowData[i] = vector.vector[rowNum];
- } else if (columnVector instanceof ListColumnVector) {
- ListColumnVector vector = (ListColumnVector) columnVector;
- StringBuilder builder = new StringBuilder();
- vector.stringifyValue(builder, rowNum);
- rowData[i] = builder.toString();
- } else if (columnVector instanceof IntervalDayTimeColumnVector) {
- IntervalDayTimeColumnVector vector = (IntervalDayTimeColumnVector) columnVector;
- StringBuilder builder = new StringBuilder();
- vector.stringifyValue(builder, rowNum);
- rowData[i] = builder.toString();
- } else if (columnVector instanceof LongColumnVector) {
- LongColumnVector vector = (LongColumnVector) columnVector;
- rowData[i] = vector.vector[rowNum];
- } else if (columnVector instanceof MapColumnVector) {
- MapColumnVector vector = (MapColumnVector) columnVector;
- StringBuilder builder = new StringBuilder();
- vector.stringifyValue(builder, rowNum);
- rowData[i] = builder.toString();
- } else if (columnVector instanceof MultiValuedColumnVector) {
- MultiValuedColumnVector vector = (MultiValuedColumnVector) columnVector;
- StringBuilder builder = new StringBuilder();
- vector.stringifyValue(builder, rowNum);
- rowData[i] = builder.toString();
- } else if (columnVector instanceof StructColumnVector) {
- StructColumnVector vector = (StructColumnVector) columnVector;
- StringBuilder builder = new StringBuilder();
- vector.stringifyValue(builder, rowNum);
- rowData[i] = builder.toString();
- } else if (columnVector instanceof TimestampColumnVector) {
- TimestampColumnVector vector = (TimestampColumnVector) columnVector;
- rowData[i] = vector.time[rowNum];
- } else if (columnVector instanceof UnionColumnVector) {
- UnionColumnVector vector = (UnionColumnVector) columnVector;
- StringBuilder builder = new StringBuilder();
- vector.stringifyValue(builder, rowNum);
- rowData[i] = builder.toString();
- }
- }
- row = new TableRecord(rowData);
- } else {
- return false;
- }
- return row != null;
- }
-
- @Override
- public Record getRecord() {
- if (metaData == null) throw new RuntimeException("Must read metadata first(必须先读取metadata)");
- if (row == null) {
- throw new RuntimeException(
- "Can't get the value of the field, maybe the IO stream has been read or has been closed!(拿不到字段的值,也许IO流已读取完毕或已被关闭!)");
- }
- return row;
- }
-
- @Override
- public void close() throws IOException {
- IOUtils.closeQuietly(inputStream);
- rows.close();
- reader.close();
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetWriter.java
deleted file mode 100644
index c4809f6499..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetWriter.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset;
-
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.common.io.resultset.ResultSet;
-import org.apache.linkis.common.io.resultset.ResultSetWriter;
-import org.apache.linkis.storage.domain.Column;
-import org.apache.linkis.storage.resultset.table.TableMetaData;
-import org.apache.linkis.storage.resultset.table.TableRecord;
-import org.apache.linkis.storage.utils.OrcUtils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.orc.CompressionKind;
-import org.apache.orc.OrcFile;
-import org.apache.orc.TypeDescription;
-import org.apache.orc.Writer;
-import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch;
-
-import java.io.IOException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class OrcResultSetWriter<K extends MetaData, V extends Record>
- extends ResultSetWriter<K, V> {
- private static final Logger logger = LoggerFactory.getLogger(OrcResultSetWriter.class);
-
- private TypeDescription schema;
-
- private Writer writer;
-
- private boolean moveToWriteRow = false;
-
- private MetaData metaData = null;
-
- private final FsPath storePath;
-
- private final long maxCacheSize;
-
- private final ResultSet<K, V> resultSet;
-
- public OrcResultSetWriter(ResultSet resultSet, long maxCacheSize, FsPath storePath) {
- super(resultSet, maxCacheSize, storePath);
- this.resultSet = resultSet;
- this.maxCacheSize = maxCacheSize;
- this.storePath = storePath;
- }
-
- @Override
- public void addMetaData(MetaData metaData) throws IOException {
- if (!moveToWriteRow) {
- this.metaData = metaData;
- if (this.schema == null) {
- this.schema = TypeDescription.createStruct();
- }
- TableMetaData tableMetaData = (TableMetaData) this.metaData;
- for (Column column : tableMetaData.columns) {
- schema.addField(column.getColumnName(), OrcUtils.dataTypeToOrcType(column.getDataType()));
- }
- moveToWriteRow = true;
- if (writer == null) {
- writer =
- OrcFile.createWriter(
- new Path(storePath.getPath()),
- OrcFile.writerOptions(new Configuration())
- .setSchema(schema)
- .compress(CompressionKind.ZLIB)
- .version(OrcFile.Version.V_0_12));
- }
- }
- }
-
- @Override
- public void addRecord(Record record) {
- if (moveToWriteRow) {
- TableRecord tableRecord = (TableRecord) record;
- TableMetaData tableMetaData = (TableMetaData) metaData;
- try {
- Object[] row = tableRecord.row;
- VectorizedRowBatch batch = schema.createRowBatch();
- int rowCount = batch.size++;
-
- for (int i = 0; i < row.length; i++) {
- OrcUtils.setColumn(
- rowCount, batch.cols[i], tableMetaData.columns[i].getDataType(), row[i]);
- if (batch.size == batch.getMaxSize()) {
- writer.addRowBatch(batch);
- batch.reset();
- }
- }
- writer.addRowBatch(batch);
-
- } catch (IOException e) {
- logger.warn("addMetaDataAndRecordString failed", e);
- }
- }
- }
-
- @Override
- public FsPath toFSPath() {
- return storePath;
- }
-
- @Override
- public String toString() {
- return storePath.getSchemaPath();
- }
-
- @Override
- public void addMetaDataAndRecordString(String content) {}
-
- @Override
- public void addRecordString(String content) {}
-
- @Override
- public void close() throws IOException {
- writer.close();
- }
-
- @Override
- public void flush() throws IOException {}
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetReader.java
deleted file mode 100644
index c09804294d..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetReader.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset;
-
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.common.io.resultset.ResultSet;
-import org.apache.linkis.common.io.resultset.ResultSetReader;
-import org.apache.linkis.storage.domain.Column;
-import org.apache.linkis.storage.domain.DataType;
-import org.apache.linkis.storage.resultset.table.TableMetaData;
-import org.apache.linkis.storage.resultset.table.TableRecord;
-
-import org.apache.avro.Schema;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.parquet.avro.AvroParquetReader;
-import org.apache.parquet.hadoop.ParquetReader;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ParquetResultSetReader<K extends MetaData, V extends Record>
- extends ResultSetReader<K, V> {
-
- private static final Logger logger = LoggerFactory.getLogger(ParquetResultSetReader.class);
-
- private FsPath fsPath;
-
- private final ResultSet<K, V> resultSet;
-
- private final InputStream inputStream;
-
- private MetaData metaData;
-
- private Record row;
-
- private ParquetReader<GenericRecord> parquetReader;
-
- private GenericRecord record;
-
- public ParquetResultSetReader(ResultSet resultSet, InputStream inputStream, FsPath fsPath)
- throws IOException {
- super(resultSet, inputStream);
- this.resultSet = resultSet;
- this.inputStream = inputStream;
- this.fsPath = fsPath;
- this.parquetReader =
- AvroParquetReader.<GenericRecord>builder(new Path(fsPath.getPath())).build();
- this.record = parquetReader.read();
- }
-
- @Override
- public MetaData getMetaData() {
- if (metaData == null) {
- try {
- List<Schema.Field> fields = record.getSchema().getFields();
- List<Column> columnList =
- fields.stream()
- .map(
- field ->
- new Column(
- field.name(),
- DataType.toDataType(field.schema().getType().getName()),
- ""))
- .collect(Collectors.toList());
-
- metaData = new TableMetaData(columnList.toArray(new Column[0]));
- } catch (Exception e) {
- throw new RuntimeException("Failed to read parquet schema", e);
- }
- }
- return metaData;
- }
-
- @Override
- public int skip(int recordNum) throws IOException {
- if (recordNum < 0) return -1;
-
- for (int i = recordNum; i > 0; i--) {
- try {
- this.record = parquetReader.read();
- } catch (Throwable t) {
- return recordNum - i;
- }
- }
- return recordNum;
- }
-
- @Override
- public long getPosition() throws IOException {
- throw new UnsupportedOperationException("Storeage Unsupported type: getPosition");
- }
-
- @Override
- public long available() throws IOException {
- throw new UnsupportedOperationException("Storeage Unsupported type: available");
- }
-
- @Override
- public boolean hasNext() throws IOException {
- if (metaData == null) getMetaData();
- if (record == null) return false;
- ArrayList<Object> resultList = new ArrayList<>();
- TableMetaData tableMetaData = (TableMetaData) metaData;
- int length = tableMetaData.getColumns().length;
- for (int i = 0; i < length; i++) {
- resultList.add(record.get(i));
- }
- row = new TableRecord(resultList.toArray(new Object[0]));
- if (row == null) return false;
- return record != null;
- }
-
- @Override
- public Record getRecord() {
- if (metaData == null) throw new RuntimeException("Must read metadata first(必须先读取metadata)");
- if (row == null) {
- throw new RuntimeException(
- "Can't get the value of the field, maybe the IO stream has been read or has been closed!(拿不到字段的值,也许IO流已读取完毕或已被关闭!)");
- }
- try {
- this.record = parquetReader.read();
- } catch (IOException e) {
- throw new RuntimeException("Failed to read parquet record", e);
- }
- return row;
- }
-
- @Override
- public void close() throws IOException {
- IOUtils.closeQuietly(inputStream);
- parquetReader.close();
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetWriter.java
deleted file mode 100644
index 6fbac3c8cb..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetWriter.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset;
-
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.common.io.resultset.ResultSet;
-import org.apache.linkis.common.io.resultset.ResultSetWriter;
-import org.apache.linkis.storage.domain.Column;
-import org.apache.linkis.storage.resultset.table.TableMetaData;
-import org.apache.linkis.storage.resultset.table.TableRecord;
-
-import org.apache.avro.Schema;
-import org.apache.avro.SchemaBuilder;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.fs.Path;
-import org.apache.parquet.avro.AvroParquetWriter;
-import org.apache.parquet.hadoop.ParquetFileWriter;
-import org.apache.parquet.hadoop.ParquetWriter;
-import org.apache.parquet.hadoop.metadata.CompressionCodecName;
-
-import java.io.IOException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ParquetResultSetWriter<K extends MetaData, V extends Record>
- extends ResultSetWriter<K, V> {
- private static final Logger logger = LoggerFactory.getLogger(ParquetResultSetWriter.class);
-
- private Schema schema;
-
- private ParquetWriter<GenericRecord> parquetWriter;
-
- private boolean moveToWriteRow = false;
-
- private MetaData metaData = null;
-
- private final FsPath storePath;
-
- private final long maxCacheSize;
-
- private final ResultSet<K, V> resultSet;
-
- public ParquetResultSetWriter(ResultSet resultSet, long maxCacheSize, FsPath storePath) {
- super(resultSet, maxCacheSize, storePath);
- this.resultSet = resultSet;
- this.maxCacheSize = maxCacheSize;
- this.storePath = storePath;
- }
-
- @Override
- public void addMetaData(MetaData metaData) throws IOException {
- if (!moveToWriteRow) {
- this.metaData = metaData;
- SchemaBuilder.FieldAssembler<Schema> fieldAssembler = SchemaBuilder.record("linkis").fields();
- TableMetaData tableMetaData = (TableMetaData) this.metaData;
- for (Column column : tableMetaData.columns) {
- fieldAssembler
- .name(column.getColumnName().replaceAll("\\.", "_").replaceAll("[^a-zA-Z0-9_]", ""))
- .doc(column.getComment())
- .type(column.getDataType().getTypeName().toLowerCase())
- .noDefault();
- }
- schema = fieldAssembler.endRecord();
- moveToWriteRow = true;
- if (parquetWriter == null) {
- parquetWriter =
- AvroParquetWriter.<GenericRecord>builder(new Path(storePath.getPath()))
- .withSchema(schema)
- .withCompressionCodec(CompressionCodecName.SNAPPY)
- .withWriteMode(ParquetFileWriter.Mode.OVERWRITE)
- .build();
- }
- }
- }
-
- @Override
- public void addRecord(Record record) {
- if (moveToWriteRow) {
- TableRecord tableRecord = (TableRecord) record;
- try {
- Object[] row = tableRecord.row;
- GenericRecord genericRecord = new GenericData.Record(schema);
- for (int i = 0; i < row.length; i++) {
- genericRecord.put(schema.getFields().get(i).name(), row[i]);
- }
- parquetWriter.write(genericRecord);
- } catch (IOException e) {
- logger.warn("addMetaDataAndRecordString failed", e);
- }
- }
- }
-
- @Override
- public FsPath toFSPath() {
- return storePath;
- }
-
- @Override
- public String toString() {
- return storePath.getSchemaPath();
- }
-
- @Override
- public void addMetaDataAndRecordString(String content) {}
-
- @Override
- public void addRecordString(String content) {}
-
- @Override
- public void close() throws IOException {
- parquetWriter.close();
- }
-
- @Override
- public void flush() throws IOException {}
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultMetaData.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultMetaData.java
deleted file mode 100644
index 04f835ac01..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultMetaData.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset;
-
-import org.apache.linkis.common.io.MetaData;
-
-public interface ResultMetaData extends MetaData {}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultRecord.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultRecord.java
deleted file mode 100644
index ce2686099b..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultRecord.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset;
-
-import org.apache.linkis.common.io.Record;
-
-public interface ResultRecord extends Record {}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetFactory.java
deleted file mode 100644
index ed65cea16c..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetFactory.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset;
-
-import org.apache.linkis.common.io.*;
-import org.apache.linkis.common.io.resultset.ResultSet;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-public interface ResultSetFactory {
-
- String TEXT_TYPE = "1";
- String TABLE_TYPE = "2";
- String IO_TYPE = "3";
- String PICTURE_TYPE = "4";
- String HTML_TYPE = "5";
-
- /** TODO 修改为注册形式,并修改ResultSet的getResultType逻辑 Result set corresponding type record(结果集对应类型记录) */
- Map<String, String> resultSetType =
- new LinkedHashMap<String, String>() {
- {
- put(TEXT_TYPE, "TEXT");
- put(TABLE_TYPE, "TABLE");
- put(IO_TYPE, "IO");
- put(PICTURE_TYPE, "PICTURE");
- put(HTML_TYPE, "HTML");
- }
- };
-
- DefaultResultSetFactory factory = new DefaultResultSetFactory();
-
- static ResultSetFactory getInstance() {
- return factory;
- }
-
- ResultSet<? extends MetaData, ? extends Record> getResultSetByType(String resultSetType);
-
- ResultSet<? extends MetaData, ? extends Record> getResultSetByPath(FsPath fsPath);
-
- ResultSet<? extends MetaData, ? extends Record> getResultSetByPath(FsPath fsPath, Fs fs);
-
- ResultSet<? extends MetaData, ? extends Record> getResultSetByContent(String content);
-
- boolean exists(String resultSetType);
-
- boolean isResultSetPath(String path);
-
- boolean isResultSet(String content);
-
- ResultSet<? extends MetaData, ? extends Record> getResultSet(String output);
-
- ResultSet<? extends MetaData, ? extends Record> getResultSetByPath(
- FsPath fsPath, String proxyUser);
-
- ResultSet<? extends MetaData, ? extends Record> getResultSet(String output, String proxyUser);
-
- String[] getResultSetType();
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java
deleted file mode 100644
index bf46f49f9b..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset;
-
-import org.apache.linkis.common.io.Fs;
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.common.io.resultset.ResultSet;
-import org.apache.linkis.common.io.resultset.ResultSetReader;
-import org.apache.linkis.storage.FSFactory;
-import org.apache.linkis.storage.conf.LinkisStorageConf;
-import org.apache.linkis.storage.domain.Dolphin;
-import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary;
-import org.apache.linkis.storage.exception.StorageErrorCode;
-import org.apache.linkis.storage.exception.StorageReadException;
-import org.apache.linkis.storage.exception.StorageWarnException;
-import org.apache.linkis.storage.resultset.table.TableMetaData;
-import org.apache.linkis.storage.resultset.table.TableRecord;
-import org.apache.linkis.storage.resultset.table.TableResultSet;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ResultSetReaderFactory {
- private static final Logger logger = LoggerFactory.getLogger(ResultSetReaderFactory.class);
-
- public static <K extends MetaData, V extends Record> ResultSetReader getResultSetReader(
- ResultSet<K, V> resultSet, InputStream inputStream, FsPath fsPath) {
- ResultSetReader<K, V> resultSetReader = null;
- if (fsPath.getPath().endsWith(Dolphin.DOLPHIN_FILE_SUFFIX)) {
- resultSetReader = new StorageResultSetReader<>(resultSet, inputStream);
- } else if (fsPath.getPath().endsWith(LinkisStorageConf.PARQUET_FILE_SUFFIX)) {
- try {
- resultSetReader = new ParquetResultSetReader<>(resultSet, inputStream, fsPath);
- } catch (IOException e) {
- throw new StorageReadException(
- StorageErrorCode.READ_PARQUET_FAILED.getCode(),
- StorageErrorCode.READ_PARQUET_FAILED.getMessage(),
- e);
- }
- } else if (fsPath.getPath().endsWith(LinkisStorageConf.ORC_FILE_SUFFIX)) {
- try {
- resultSetReader = new OrcResultSetReader<>(resultSet, inputStream, fsPath);
- } catch (IOException e) {
- throw new StorageReadException(
- StorageErrorCode.READ_ORC_FAILED.getCode(),
- StorageErrorCode.READ_ORC_FAILED.getMessage(),
- e);
- }
- }
- return resultSetReader;
- }
-
- public static <K extends MetaData, V extends Record> ResultSetReader getResultSetReader(
- ResultSet<K, V> resultSet, String value) {
- return new StorageResultSetReader<>(resultSet, value);
- }
-
- public static ResultSetReader getResultSetReader(String res) throws IOException {
- ResultSetFactory rsFactory = ResultSetFactory.getInstance();
- if (rsFactory.isResultSet(res)) {
- ResultSet<? extends MetaData, ? extends Record> resultSet = rsFactory.getResultSet(res);
- return ResultSetReaderFactory.getResultSetReader(resultSet, res);
- } else {
- FsPath resPath = new FsPath(res);
- ResultSet<? extends MetaData, ? extends Record> resultSet =
- rsFactory.getResultSetByPath(resPath);
- Fs fs = FSFactory.getFs(resPath);
- fs.init(null);
- ResultSetReader reader =
- ResultSetReaderFactory.getResultSetReader(resultSet, fs.read(resPath), resPath);
- if (reader instanceof StorageResultSetReader) {
- ((StorageResultSetReader<?, ?>) reader).setFs(fs);
- }
- return (StorageResultSetReader<?, ?>) reader;
- }
- }
-
- public static ResultSetReader getTableResultReader(String res) {
- ResultSetFactory rsFactory = ResultSetFactory.getInstance();
- if (rsFactory.isResultSet(res)) {
- ResultSet<?, ?> resultSet = rsFactory.getResultSet(res);
- if (!ResultSetFactory.TABLE_TYPE.equals(resultSet.resultSetType())) {
- throw new StorageWarnException(
- LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED.getErrorCode(),
- LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED.getErrorDesc());
- }
- return ResultSetReaderFactory.<TableMetaData, TableRecord>getResultSetReader(
- (TableResultSet) resultSet, res);
- } else {
- FsPath resPath = new FsPath(res);
- ResultSet<?, ?> resultSet = rsFactory.getResultSetByPath(resPath);
- if (!ResultSetFactory.TABLE_TYPE.equals(resultSet.resultSetType())) {
- throw new StorageWarnException(
- LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED.getErrorCode(),
- LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED.getErrorDesc());
- }
-
- Fs fs = FSFactory.getFs(resPath);
- logger.info("Try to init Fs with path:{}", resPath.getPath());
- try {
- fs.init(null);
- InputStream read = fs.read(resPath);
-
- return ResultSetReaderFactory.<TableMetaData, TableRecord>getResultSetReader(
- (TableResultSet) resultSet, read, resPath);
- } catch (IOException e) {
- throw new StorageWarnException(
- LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED.getErrorCode(),
- LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED.getErrorDesc());
- }
- }
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetWriterFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetWriterFactory.java
deleted file mode 100644
index 9cb1999367..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetWriterFactory.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset;
-
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.common.io.resultset.ResultSet;
-import org.apache.linkis.common.io.resultset.ResultSetReader;
-import org.apache.linkis.common.io.resultset.ResultSetWriter;
-import org.apache.linkis.storage.conf.LinkisStorageConf;
-import org.apache.linkis.storage.resultset.table.TableResultSet;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ResultSetWriterFactory {
- private static final Logger logger = LoggerFactory.getLogger(ResultSetWriterFactory.class);
-
- public static <K extends MetaData, V extends Record> ResultSetWriter<K, V> getResultSetWriter(
- ResultSet<K, V> resultSet, long maxCacheSize, FsPath storePath) {
- String engineResultType = LinkisStorageConf.ENGINE_RESULT_TYPE;
- ResultSetWriter<K, V> writer = null;
- if (engineResultType.equals(LinkisStorageConf.PARQUET) && resultSet instanceof TableResultSet) {
- writer = new ParquetResultSetWriter<>(resultSet, maxCacheSize, storePath);
- } else if (engineResultType.equals(LinkisStorageConf.ORC)
- && resultSet instanceof TableResultSet) {
- writer = new OrcResultSetWriter<>(resultSet, maxCacheSize, storePath);
- } else {
- writer = new StorageResultSetWriter<>(resultSet, maxCacheSize, storePath);
- }
- return writer;
- }
-
- public static <K extends MetaData, V extends Record> ResultSetWriter<K, V> getResultSetWriter(
- ResultSet<K, V> resultSet, long maxCacheSize, FsPath storePath, String proxyUser) {
- String engineResultType = LinkisStorageConf.ENGINE_RESULT_TYPE;
- ResultSetWriter<K, V> writer = null;
- if (engineResultType.equals(LinkisStorageConf.PARQUET) && resultSet instanceof TableResultSet) {
- writer = new ParquetResultSetWriter<>(resultSet, maxCacheSize, storePath);
- } else if (engineResultType.equals(LinkisStorageConf.ORC)
- && resultSet instanceof TableResultSet) {
- writer = new OrcResultSetWriter<>(resultSet, maxCacheSize, storePath);
- } else {
- writer = new StorageResultSetWriter<>(resultSet, maxCacheSize, storePath);
- StorageResultSetWriter storageResultSetWriter = (StorageResultSetWriter) writer;
- storageResultSetWriter.setProxyUser(proxyUser);
- }
- return writer;
- }
-
- public static Record[] getRecordByWriter(
- ResultSetWriter<? extends MetaData, ? extends Record> writer, long limit) throws IOException {
- String res = writer.toString();
- return getRecordByRes(res, limit);
- }
-
- public static Record[] getRecordByRes(String res, long limit) throws IOException {
- ResultSetReader reader = ResultSetReaderFactory.getResultSetReader(res);
- int count = 0;
- List<Record> records = new ArrayList<>();
- reader.getMetaData();
- while (reader.hasNext() && count < limit) {
- records.add(reader.getRecord());
- count++;
- }
- return records.toArray(new Record[0]);
- }
-
- public static Record getLastRecordByRes(String res) throws IOException {
- ResultSetReader reader = ResultSetReaderFactory.getResultSetReader(res);
- Record record = null;
- try {
- reader.getMetaData();
- while (reader.hasNext()) {
- record = reader.getRecord();
- }
- } catch (IOException e) {
- logger.warn("ResultSetWriter getLastRecordByRes failed", e);
- }
- return record;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSet.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSet.java
deleted file mode 100644
index c708f5faf6..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSet.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset;
-
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.common.io.resultset.ResultSet;
-import org.apache.linkis.storage.conf.LinkisStorageConf;
-import org.apache.linkis.storage.domain.Dolphin;
-import org.apache.linkis.storage.resultset.table.TableResultSet;
-import org.apache.linkis.storage.utils.StorageConfiguration;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class StorageResultSet<K extends MetaData, V extends Record>
- implements ResultSet<K, V> {
-
- private static final Logger logger = LoggerFactory.getLogger(StorageResultSet.class);
-
- private byte[] resultHeaderBytes = null;
-
- {
- byte[] arr2 = Dolphin.getIntBytes(Integer.parseInt(resultSetType()));
- byte[] mergedArray = new byte[Dolphin.MAGIC_BYTES.length + arr2.length];
- System.arraycopy(Dolphin.MAGIC_BYTES, 0, mergedArray, 0, Dolphin.MAGIC_BYTES.length);
- System.arraycopy(arr2, 0, mergedArray, Dolphin.MAGIC_BYTES.length, arr2.length);
- resultHeaderBytes = mergedArray;
- }
-
- @Override
- public String charset() {
- return StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue();
- }
-
- @Override
- public FsPath getResultSetPath(FsPath parentDir, String fileName) {
- String engineResultType = LinkisStorageConf.ENGINE_RESULT_TYPE;
- String fileSuffix = Dolphin.DOLPHIN_FILE_SUFFIX;
- if (engineResultType.equals(LinkisStorageConf.PARQUET) && this instanceof TableResultSet) {
- fileSuffix = LinkisStorageConf.PARQUET_FILE_SUFFIX;
- } else if (engineResultType.equals(LinkisStorageConf.ORC) && this instanceof TableResultSet) {
- fileSuffix = LinkisStorageConf.ORC_FILE_SUFFIX;
- }
- final String path =
- parentDir.getPath().endsWith("/")
- ? parentDir.getUriString() + fileName + fileSuffix
- : parentDir.getUriString() + "/" + fileName + fileSuffix;
- logger.info("Get result set path: {}", path);
- return new FsPath(path);
- }
-
- @Override
- public byte[] getResultSetHeader() {
- return resultHeaderBytes;
- }
-
- @Override
- public boolean belongToPath(String path) {
- return path.endsWith(Dolphin.DOLPHIN_FILE_SUFFIX);
- }
-
- @Override
- public boolean belongToResultSet(String content) {
- try {
- return Dolphin.getType(content).equals(resultSetType());
- } catch (Exception e) {
- logger.info("Wrong result set: ", e);
- return false;
- }
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetReader.java
deleted file mode 100644
index c0222cc848..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetReader.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset;
-
-import org.apache.linkis.common.io.Fs;
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.common.io.resultset.ResultDeserializer;
-import org.apache.linkis.common.io.resultset.ResultSet;
-import org.apache.linkis.common.io.resultset.ResultSetReader;
-import org.apache.linkis.storage.domain.Dolphin;
-import org.apache.linkis.storage.exception.StorageWarnException;
-import org.apache.linkis.storage.utils.StorageUtils;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class StorageResultSetReader<K extends MetaData, V extends Record>
- extends ResultSetReader<K, V> {
-
- private static final Logger logger = LoggerFactory.getLogger(StorageResultSetReader.class);
-
- private final ResultSet<K, V> resultSet;
- private final InputStream inputStream;
- private final ResultDeserializer<K, V> deserializer;
- private K metaData;
- private Record row;
- private int colCount = 0;
- private int rowCount = 0;
- private Fs fs;
-
- private final int READ_CACHE = 1024;
-
- public StorageResultSetReader(ResultSet<K, V> resultSet, InputStream inputStream) {
- super(resultSet, inputStream);
- this.resultSet = resultSet;
- this.inputStream = inputStream;
- this.deserializer = resultSet.createResultSetDeserializer();
- }
-
- public StorageResultSetReader(ResultSet<K, V> resultSet, String value) {
- this(resultSet, new ByteArrayInputStream(value.getBytes(Dolphin.CHAR_SET)));
- }
-
- public void init() throws IOException {
- String resType = Dolphin.getType(inputStream);
- if (!StringUtils.equals(resultSet.resultSetType(), resType)) {
- throw new RuntimeException(
- "File type does not match(文件类型不匹配): "
- + ResultSetFactory.resultSetType.getOrDefault(resType, "TABLE"));
- }
- }
-
- public byte[] readLine() {
- int rowLen = 0;
- try {
- rowLen = Dolphin.readInt(inputStream);
- } catch (StorageWarnException | IOException e) {
- logger.info("Read finished(读取完毕)");
- return null;
- }
-
- int len = 0;
- byte[] rowBuffer = null;
- try {
- rowBuffer = new byte[rowLen];
- len = StorageUtils.readBytes(inputStream, rowBuffer, rowLen);
- } catch (OutOfMemoryError error) {
- logger.error("Result set read oom, read size {} Byte", rowLen);
- throw new RuntimeException(error);
- }
- if (len != rowLen) {
- throw new RuntimeException(
- "Can't get the value of the field, maybe the IO stream has been read or has been closed!(拿不到字段的值,也许IO流已读取完毕或已被关闭!)");
- }
- rowCount++;
- return rowBuffer;
- }
-
- @Override
- public Record getRecord() {
- if (metaData == null) throw new RuntimeException("Must read metadata first(必须先读取metadata)");
- if (row == null) {
- throw new RuntimeException(
- "Can't get the value of the field, maybe the IO stream has been read or has been closed!(拿不到字段的值,也许IO流已读取完毕或已被关闭!)");
- }
- return row;
- }
-
- public void setFs(Fs fs) {
- this.fs = fs;
- }
-
- public Fs getFs() {
- return fs;
- }
-
- @Override
- public MetaData getMetaData() {
- if (metaData == null) {
- try {
- init();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- metaData = deserializer.createMetaData(readLine());
- return metaData;
- }
-
- @Override
- public int skip(int recordNum) throws IOException {
- if (recordNum < 0) return -1;
-
- if (metaData == null) getMetaData();
- for (int i = recordNum; i > 0; i--) {
- try {
- inputStream.skip(Dolphin.readInt(inputStream));
- } catch (Throwable t) {
- return recordNum - i;
- }
- }
- return recordNum;
- }
-
- @Override
- public long getPosition() throws IOException {
- return rowCount;
- }
-
- @Override
- public boolean hasNext() throws IOException {
- if (metaData == null) getMetaData();
- byte[] line = readLine();
- if (line == null) return false;
- row = deserializer.createRecord(line);
- if (row == null) return false;
- return true;
- }
-
- @Override
- public long available() throws IOException {
- return inputStream.available();
- }
-
- @Override
- public void close() throws IOException {
- IOUtils.closeQuietly(inputStream);
- if (this.fs != null) {
- this.fs.close();
- }
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetWriter.java
deleted file mode 100644
index 230c68301c..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetWriter.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset;
-
-import org.apache.linkis.common.io.Fs;
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.common.io.resultset.ResultSerializer;
-import org.apache.linkis.common.io.resultset.ResultSet;
-import org.apache.linkis.common.io.resultset.ResultSetWriter;
-import org.apache.linkis.storage.FSFactory;
-import org.apache.linkis.storage.conf.LinkisStorageConf;
-import org.apache.linkis.storage.domain.Dolphin;
-import org.apache.linkis.storage.exception.StorageErrorException;
-import org.apache.linkis.storage.utils.FileSystemUtils;
-import org.apache.linkis.storage.utils.StorageUtils;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.linkis.storage.exception.StorageErrorCode.FS_ERROR;
-
-public class StorageResultSetWriter<K extends MetaData, V extends Record>
- extends ResultSetWriter<K, V> {
- private static final Logger logger = LoggerFactory.getLogger(StorageResultSetWriter.class);
-
- private final ResultSet<K, V> resultSet;
- private final long maxCacheSize;
- private final FsPath storePath;
-
- private final ResultSerializer serializer;
- private boolean moveToWriteRow = false;
- private OutputStream outputStream = null;
- private int rowCount = 0;
- private final List<Byte> buffer = new ArrayList<Byte>();
- private Fs fs = null;
- private MetaData rMetaData = null;
- private String proxyUser = StorageUtils.getJvmUser();
- private boolean fileCreated = false;
- private boolean closed = false;
- private final Object WRITER_LOCK_CREATE = new Object();
- private final Object WRITER_LOCK_CLOSE = new Object();
-
- public StorageResultSetWriter(ResultSet<K, V> resultSet, long maxCacheSize, FsPath storePath) {
- super(resultSet, maxCacheSize, storePath);
- this.resultSet = resultSet;
- this.maxCacheSize = maxCacheSize;
- this.storePath = storePath;
-
- this.serializer = resultSet.createResultSetSerializer();
- }
-
- public MetaData getMetaData() {
- return rMetaData;
- }
-
- public void setProxyUser(String proxyUser) {
- this.proxyUser = proxyUser;
- }
-
- public boolean isEmpty() {
- return rMetaData == null && buffer.size() <= Dolphin.FILE_EMPTY;
- }
-
- public void init() {
- try {
- writeLine(resultSet.getResultSetHeader(), true);
- } catch (IOException e) {
- logger.warn("StorageResultSetWriter init failed", e);
- }
- }
-
- public void createNewFile() {
- if (!fileCreated) {
- synchronized (WRITER_LOCK_CREATE) {
- if (!fileCreated) {
- if (storePath != null && outputStream == null) {
- logger.info("Try to create a new file:{}, with proxy user:{}", storePath, proxyUser);
- fs = FSFactory.getFsByProxyUser(storePath, proxyUser);
- try {
- fs.init(null);
- FileSystemUtils.createNewFile(storePath, proxyUser, true);
- outputStream = fs.write(storePath, true);
- } catch (Exception e) {
- throw new StorageErrorException(
- FS_ERROR.getCode(), "StorageResultSetWriter createNewFile failed", e);
- }
- logger.info("Succeed to create a new file:{}", storePath);
- fileCreated = true;
- }
- }
- }
- } else if (storePath != null && outputStream == null) {
- logger.warn("outputStream had been set null, but createNewFile() was called again.");
- }
- }
-
- public void writeLine(byte[] bytes, boolean cache) throws IOException {
- if (closed) {
- logger.warn("the writer had been closed, but writeLine() was still called.");
- return;
- }
- if (bytes.length > LinkisStorageConf.ROW_BYTE_MAX_LEN) {
- throw new IOException(
- String.format(
- "A single row of data cannot exceed %s", LinkisStorageConf.ROW_BYTE_MAX_LEN_STR));
- }
- if (buffer.size() > maxCacheSize && !cache) {
- if (outputStream == null) {
- createNewFile();
- }
- flush();
- outputStream.write(bytes);
- } else {
- for (byte b : bytes) {
- buffer.add(b);
- }
- }
- }
-
- @Override
- public String toString() {
- if (outputStream == null) {
- if (isEmpty()) {
- return "";
- }
-
- byte[] byteArray = getBytes();
- return new String(byteArray, Dolphin.CHAR_SET);
- }
- return storePath.getSchemaPath();
- }
-
- private byte[] getBytes() {
- byte[] byteArray = new byte[buffer.size()];
- for (int i = 0; i < buffer.size(); i++) {
- byteArray[i] = buffer.get(i);
- }
- return byteArray;
- }
-
- @Override
- public FsPath toFSPath() {
- return storePath;
- }
-
- @Override
- public void addMetaDataAndRecordString(String content) {
- if (!moveToWriteRow) {
- byte[] bytes = content.getBytes(Dolphin.CHAR_SET);
- try {
- writeLine(bytes, false);
- } catch (IOException e) {
- logger.warn("addMetaDataAndRecordString failed", e);
- }
- }
- moveToWriteRow = true;
- }
-
- @Override
- public void addRecordString(String content) {}
-
- @Override
- public void addMetaData(MetaData metaData) throws IOException {
- if (!moveToWriteRow) {
- rMetaData = metaData;
- init();
- if (metaData == null) {
- writeLine(serializer.metaDataToBytes(metaData), true);
- } else {
- writeLine(serializer.metaDataToBytes(metaData), false);
- }
- moveToWriteRow = true;
- }
- }
-
- @Override
- public void addRecord(Record record) {
- if (moveToWriteRow) {
- rowCount++;
- try {
- writeLine(serializer.recordToBytes(record), false);
- } catch (IOException e) {
- logger.warn("addMetaDataAndRecordString failed", e);
- }
- }
- }
-
- public void closeFs() {
- if (fs != null) {
- IOUtils.closeQuietly(fs);
- fs = null;
- }
- }
-
- @Override
- public void close() {
- if (closed) {
- logger.warn("the writer had been closed, but close() was still called.");
- return;
- }
- synchronized (WRITER_LOCK_CLOSE) {
- if (!closed) {
- closed = true;
- } else {
- return;
- }
- }
- try {
- if (outputStream != null) {
- flush();
- }
- } finally {
- if (outputStream != null) {
- IOUtils.closeQuietly(outputStream);
- outputStream = null;
- }
- closeFs();
- }
- }
-
- @Override
- public void flush() {
- createNewFile();
- if (outputStream != null) {
- try {
- if (!buffer.isEmpty()) {
- outputStream.write(getBytes());
- buffer.clear();
- }
- if (outputStream instanceof HdfsDataOutputStream) {
- ((HdfsDataOutputStream) outputStream).hflush();
- } else {
- outputStream.flush();
- }
- } catch (IOException e) {
- logger.warn("Error encountered when flush result set", e);
- }
- }
- if (closed && logger.isDebugEnabled()) {
- logger.debug("the writer had been closed, but flush() was still called.");
- }
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/html/HtmlResultSet.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/html/HtmlResultSet.java
deleted file mode 100644
index 00c0e7b2a7..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/html/HtmlResultSet.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.html;
-
-import org.apache.linkis.common.io.resultset.ResultDeserializer;
-import org.apache.linkis.common.io.resultset.ResultSerializer;
-import org.apache.linkis.storage.LineMetaData;
-import org.apache.linkis.storage.LineRecord;
-import org.apache.linkis.storage.resultset.ResultSetFactory;
-import org.apache.linkis.storage.resultset.StorageResultSet;
-import org.apache.linkis.storage.resultset.txt.TextResultDeserializer;
-import org.apache.linkis.storage.resultset.txt.TextResultSerializer;
-
-import java.io.Serializable;
-
-public class HtmlResultSet extends StorageResultSet<LineMetaData, LineRecord>
- implements Serializable {
-
- @Override
- public String resultSetType() {
- return ResultSetFactory.HTML_TYPE;
- }
-
- @Override
- public ResultSerializer createResultSetSerializer() {
- return new TextResultSerializer();
- }
-
- @Override
- public ResultDeserializer<LineMetaData, LineRecord> createResultSetDeserializer() {
- return new TextResultDeserializer();
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOMetaData.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOMetaData.java
deleted file mode 100644
index bc450b5ea4..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOMetaData.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.io;
-
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.storage.resultset.ResultMetaData;
-
-public class IOMetaData implements ResultMetaData {
-
- public int off;
- public int len;
-
- public IOMetaData(int off, int len) {
- this.off = off;
- this.len = len;
- }
-
- @Override
- public MetaData cloneMeta() {
- return new IOMetaData(off, len);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IORecord.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IORecord.java
deleted file mode 100644
index 90bcbe9da8..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IORecord.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.io;
-
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.storage.resultset.ResultRecord;
-
-public class IORecord implements ResultRecord {
-
- public byte[] value;
-
- public IORecord(byte[] value) {
- this.value = value;
- }
-
- @Override
- public Record cloneRecord() {
- return new IORecord(value);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultDeserializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultDeserializer.java
deleted file mode 100644
index 02d83aa7bb..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultDeserializer.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.io;
-
-import org.apache.linkis.common.io.resultset.ResultDeserializer;
-import org.apache.linkis.storage.domain.Dolphin;
-
-import org.apache.commons.codec.binary.Base64;
-
-public class IOResultDeserializer extends ResultDeserializer<IOMetaData, IORecord> {
-
- @Override
- public IOMetaData createMetaData(byte[] bytes) {
- String[] values = Dolphin.getString(bytes, 0, bytes.length).split(Dolphin.COL_SPLIT);
- return new IOMetaData(Integer.parseInt(values[0]), Integer.parseInt(values[1]));
- }
-
- @Override
- public IORecord createRecord(byte[] bytes) {
- return new IORecord(Base64.decodeBase64(Dolphin.getString(bytes, 0, bytes.length)));
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultSerializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultSerializer.java
deleted file mode 100644
index 2401e361a5..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultSerializer.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.io;
-
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.common.io.resultset.ResultSerializer;
-import org.apache.linkis.storage.domain.Dolphin;
-import org.apache.linkis.storage.utils.StorageUtils;
-
-import org.apache.commons.codec.binary.Base64;
-
-public class IOResultSerializer extends ResultSerializer {
-
- @Override
- public byte[] metaDataToBytes(MetaData metaData) {
- IOMetaData ioMetaData = (IOMetaData) metaData;
- return lineToBytes(ioMetaData.off + Dolphin.COL_SPLIT + ioMetaData.len);
- }
-
- @Override
- public byte[] recordToBytes(Record record) {
- IORecord ioRecord = (IORecord) record;
- return lineToBytes(Base64.encodeBase64String(ioRecord.value));
- }
-
- private byte[] lineToBytes(String value) {
- byte[] bytes = value == null ? Dolphin.NULL_BYTES : Dolphin.getBytes(value);
- byte[] intBytes = Dolphin.getIntBytes(bytes.length);
- return StorageUtils.mergeByteArrays(intBytes, bytes);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultSet.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultSet.java
deleted file mode 100644
index 67f8e76904..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultSet.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.io;
-
-import org.apache.linkis.common.io.resultset.ResultDeserializer;
-import org.apache.linkis.common.io.resultset.ResultSerializer;
-import org.apache.linkis.storage.resultset.ResultSetFactory;
-import org.apache.linkis.storage.resultset.StorageResultSet;
-
-import java.io.Serializable;
-
-public class IOResultSet extends StorageResultSet<IOMetaData, IORecord> implements Serializable {
-
- @Override
- public String resultSetType() {
- return ResultSetFactory.IO_TYPE;
- }
-
- @Override
- public ResultSerializer createResultSetSerializer() {
- return new IOResultSerializer();
- }
-
- @Override
- public ResultDeserializer<IOMetaData, IORecord> createResultSetDeserializer() {
- return new IOResultDeserializer();
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/picture/PictureResultSet.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/picture/PictureResultSet.java
deleted file mode 100644
index 5e73592a7b..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/picture/PictureResultSet.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.picture;
-
-import org.apache.linkis.common.io.resultset.ResultDeserializer;
-import org.apache.linkis.common.io.resultset.ResultSerializer;
-import org.apache.linkis.storage.LineMetaData;
-import org.apache.linkis.storage.LineRecord;
-import org.apache.linkis.storage.resultset.ResultSetFactory;
-import org.apache.linkis.storage.resultset.StorageResultSet;
-import org.apache.linkis.storage.resultset.txt.TextResultDeserializer;
-import org.apache.linkis.storage.resultset.txt.TextResultSerializer;
-
-import java.io.Serializable;
-
-public class PictureResultSet extends StorageResultSet<LineMetaData, LineRecord>
- implements Serializable {
-
- @Override
- public String resultSetType() {
- return ResultSetFactory.PICTURE_TYPE;
- }
-
- @Override
- public ResultSerializer createResultSetSerializer() {
- return new TextResultSerializer();
- }
-
- @Override
- public ResultDeserializer<LineMetaData, LineRecord> createResultSetDeserializer() {
- return new TextResultDeserializer();
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableMetaData.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableMetaData.java
deleted file mode 100644
index 429ab33c82..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableMetaData.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.table;
-
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.storage.domain.Column;
-import org.apache.linkis.storage.resultset.ResultMetaData;
-
-public class TableMetaData implements ResultMetaData {
-
- public Column[] columns;
-
- public TableMetaData(Column[] columns) {
- this.columns = columns;
- }
-
- public Column[] getColumns() {
- return columns;
- }
-
- public void setColumns(Column[] columns) {
- this.columns = columns;
- }
-
- @Override
- public MetaData cloneMeta() {
- return new TableMetaData(columns.clone());
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableRecord.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableRecord.java
deleted file mode 100644
index c8270714ba..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableRecord.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.table;
-
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.storage.resultset.ResultRecord;
-
-public class TableRecord implements ResultRecord {
-
- public final Object[] row;
-
- public TableRecord(Object[] row) {
- this.row = row;
- }
-
- @Override
- public Record cloneRecord() {
- return new TableRecord(row.clone());
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultDeserializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultDeserializer.java
deleted file mode 100644
index 7e1d6c35fe..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultDeserializer.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.table;
-
-import org.apache.linkis.common.io.resultset.ResultDeserializer;
-import org.apache.linkis.storage.domain.Column;
-import org.apache.linkis.storage.domain.DataType;
-import org.apache.linkis.storage.domain.Dolphin;
-import org.apache.linkis.storage.exception.StorageWarnException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.PARSING_METADATA_FAILED;
-
-public class TableResultDeserializer extends ResultDeserializer<TableMetaData, TableRecord> {
- private static final Logger logger = LoggerFactory.getLogger(TableResultDeserializer.class);
-
- private TableMetaData metaData;
-
- @Override
- public TableMetaData createMetaData(byte[] bytes) {
- int colByteLen = Integer.parseInt(Dolphin.getString(bytes, 0, Dolphin.INT_LEN));
- String colString = Dolphin.getString(bytes, Dolphin.INT_LEN, colByteLen);
- String[] colArray =
- colString.endsWith(Dolphin.COL_SPLIT)
- ? colString.substring(0, colString.length() - 1).split(Dolphin.COL_SPLIT)
- : colString.split(Dolphin.COL_SPLIT);
- int index = Dolphin.INT_LEN + colByteLen;
- if (colArray.length % 3 != 0) {
- throw new StorageWarnException(
- PARSING_METADATA_FAILED.getErrorCode(), PARSING_METADATA_FAILED.getErrorDesc());
- }
- List<Column> columns = new ArrayList<>();
- for (int i = 0; i < colArray.length; i += 3) {
- int len = Integer.parseInt(colArray[i]);
- String colName = Dolphin.toStringValue(Dolphin.getString(bytes, index, len));
- index += len;
- len = Integer.parseInt(colArray[i + 1]);
- String colType = Dolphin.toStringValue(Dolphin.getString(bytes, index, len));
- index += len;
- len = Integer.parseInt(colArray[i + 2]);
- String colComment = Dolphin.toStringValue(Dolphin.getString(bytes, index, len));
- index += len;
- columns.add(new Column(colName, DataType.toDataType(colType), colComment));
- }
- metaData = new TableMetaData(columns.toArray(new Column[0]));
- return metaData;
- }
-
- /**
- * colByteLen:All column fields are long(所有列字段长 记录的长度) colString:Obtain column
- * length(获得列长):10,20,21 colArray:Column length array(列长数组) Get data by column length(通过列长获得数据)
- *
- * @param bytes
- * @return
- */
- @Override
- public TableRecord createRecord(byte[] bytes) {
- int colByteLen = Integer.parseInt(Dolphin.getString(bytes, 0, Dolphin.INT_LEN));
- String colString = Dolphin.getString(bytes, Dolphin.INT_LEN, colByteLen);
- String[] colArray;
- if (colString.endsWith(Dolphin.COL_SPLIT)) {
- colArray = colString.substring(0, colString.length() - 1).split(Dolphin.COL_SPLIT);
- } else {
- colArray = colString.split(Dolphin.COL_SPLIT);
- }
- int index = Dolphin.INT_LEN + colByteLen;
- Object[] data = new Object[colArray.length];
- for (int i = 0; i < colArray.length; i++) {
- int len = Integer.parseInt(colArray[i]);
- String res = Dolphin.getString(bytes, index, len);
- index += len;
- if (i >= metaData.columns.length) {
- data[i] = res;
- } else {
- data[i] = DataType.toValue(metaData.columns[i].getDataType(), res);
- }
- }
- return new TableRecord(data);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSerializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSerializer.java
deleted file mode 100644
index 5f40aa33f3..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSerializer.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.table;
-
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.common.io.resultset.ResultSerializer;
-import org.apache.linkis.storage.domain.Column;
-import org.apache.linkis.storage.domain.Dolphin;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-public class TableResultSerializer extends ResultSerializer {
-
- @Override
- public byte[] metaDataToBytes(MetaData metaData) {
- TableMetaData tableMetaData = (TableMetaData) metaData;
- Object[] objects =
- Arrays.stream(tableMetaData.columns).map(Column::toArray).flatMap(Arrays::stream).toArray();
- return lineToBytes(objects);
- }
-
- @Override
- public byte[] recordToBytes(Record record) {
- TableRecord tableRecord = (TableRecord) record;
- return lineToBytes(tableRecord.row);
- }
-
- /**
- * Convert a row of data to an array of Bytes Convert the data to byte and get the corresponding
- * total byte length to write to the file Data write format: line length (fixed length) column
- * length (fixed length) field index comma segmentation real data For example:
- * 000000004900000000116,10,3,4,5,peace1johnnwang1101true11.51 The length of the line does not
- * include its own length 将一行数据转换为Bytes的数组 对数据转换为byte,并获取相应的总byte长度写入文件 数据写入格式:行长(固定长度) 列长(固定长度)
- * 字段索引逗号分割 真实数据 如:000000004900000000116,10,3,4,5,peace1johnnwang1101true11.51 其中行长不包括自身长度
- *
- * @param line
- */
- private byte[] lineToBytes(Object[] line) {
- // Data cache(数据缓存)
- List<byte[]> dataBytes = new ArrayList<>();
- // Column cache(列缓存)
- List<byte[]> colIndex = new ArrayList<>();
- int colByteLen = 0;
- int length = 0;
- for (Object data : line) {
- byte[] bytes = data == null ? Dolphin.LINKIS_NULL_BYTES : Dolphin.getBytes(data);
- dataBytes.add(bytes);
- byte[] colBytes = Dolphin.getBytes(bytes.length);
- colIndex.add(colBytes);
- colIndex.add(Dolphin.COL_SPLIT_BYTES);
- colByteLen += colBytes.length + Dolphin.COL_SPLIT_LEN;
- length += bytes.length;
- }
- length += colByteLen + Dolphin.INT_LEN;
- return toByteArray(length, colByteLen, colIndex, dataBytes);
- }
-
- /**
- * Splice a row of data into a byte array(将一行的数据拼接成byte数组)
- *
- * @param length The total length of the line data byte, excluding its own
- * length(行数据byte总长度,不包括自身的长度)
- * @param colByteLen Record field index byte column length(记录字段索引byte的列长)
- * @param colIndex Field index, including separator comma(字段索引,包括分割符逗号)
- * @param dataBytes Byte of real data(真实数据的byte)
- * @return
- */
- public static byte[] toByteArray(
- int length, int colByteLen, List<byte[]> colIndex, List<byte[]> dataBytes) {
- List<Byte> row = new ArrayList<>();
- colIndex.addAll(dataBytes);
-
- for (byte intByte : Dolphin.getIntBytes(length)) {
- row.add(intByte);
- }
-
- for (byte colByte : Dolphin.getIntBytes(colByteLen)) {
- row.add(colByte);
- }
-
- colIndex.forEach(
- bytes -> {
- for (byte b : bytes) {
- row.add(b);
- }
- });
- byte[] result = new byte[row.size()];
- for (int i = 0; i < result.length; i++) {
- result[i] = row.get(i);
- }
- return result;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSet.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSet.java
deleted file mode 100644
index 5f356af0ca..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSet.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.table;
-
-import org.apache.linkis.common.io.resultset.ResultDeserializer;
-import org.apache.linkis.common.io.resultset.ResultSerializer;
-import org.apache.linkis.storage.resultset.ResultSetFactory;
-import org.apache.linkis.storage.resultset.StorageResultSet;
-
-import java.io.Serializable;
-
-public class TableResultSet extends StorageResultSet<TableMetaData, TableRecord>
- implements Serializable {
-
- @Override
- public String resultSetType() {
- return ResultSetFactory.TABLE_TYPE;
- }
-
- @Override
- public ResultSerializer createResultSetSerializer() {
- return new TableResultSerializer();
- }
-
- @Override
- public ResultDeserializer<TableMetaData, TableRecord> createResultSetDeserializer() {
- return new TableResultDeserializer();
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultDeserializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultDeserializer.java
deleted file mode 100644
index 3165e5af86..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultDeserializer.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.txt;
-
-import org.apache.linkis.common.io.resultset.ResultDeserializer;
-import org.apache.linkis.storage.LineMetaData;
-import org.apache.linkis.storage.LineRecord;
-import org.apache.linkis.storage.domain.Dolphin;
-
-public class TextResultDeserializer extends ResultDeserializer<LineMetaData, LineRecord> {
-
- @Override
- public LineMetaData createMetaData(byte[] bytes) {
- return new LineMetaData(Dolphin.getString(bytes, 0, bytes.length));
- }
-
- @Override
- public LineRecord createRecord(byte[] bytes) {
- return new LineRecord(Dolphin.getString(bytes, 0, bytes.length));
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultSerializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultSerializer.java
deleted file mode 100644
index 5555ad9eba..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultSerializer.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.txt;
-
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.common.io.resultset.ResultSerializer;
-import org.apache.linkis.storage.LineMetaData;
-import org.apache.linkis.storage.LineRecord;
-import org.apache.linkis.storage.domain.Dolphin;
-import org.apache.linkis.storage.utils.StorageUtils;
-
-public class TextResultSerializer extends ResultSerializer {
-
- @Override
- public byte[] metaDataToBytes(MetaData metaData) {
- if (metaData == null) {
- return lineToBytes(null);
- } else {
- LineMetaData textMetaData = (LineMetaData) metaData;
- return lineToBytes(textMetaData.getMetaData());
- }
- }
-
- @Override
- public byte[] recordToBytes(Record record) {
- LineRecord textRecord = (LineRecord) record;
- return lineToBytes(textRecord.getLine());
- }
-
- private byte[] lineToBytes(String value) {
- byte[] bytes = (value == null) ? Dolphin.NULL_BYTES : Dolphin.getBytes(value);
- return StorageUtils.mergeByteArrays(Dolphin.getIntBytes(bytes.length), bytes);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultSet.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultSet.java
deleted file mode 100644
index 19fd8f9dbc..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultSet.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.resultset.txt;
-
-import org.apache.linkis.common.io.resultset.ResultDeserializer;
-import org.apache.linkis.common.io.resultset.ResultSerializer;
-import org.apache.linkis.storage.LineMetaData;
-import org.apache.linkis.storage.LineRecord;
-import org.apache.linkis.storage.resultset.ResultSetFactory;
-import org.apache.linkis.storage.resultset.StorageResultSet;
-
-import java.io.Serializable;
-
-public class TextResultSet extends StorageResultSet<LineMetaData, LineRecord>
- implements Serializable {
-
- @Override
- public String resultSetType() {
- return ResultSetFactory.TEXT_TYPE;
- }
-
- @Override
- public ResultSerializer createResultSetSerializer() {
- return new TextResultSerializer();
- }
-
- @Override
- public ResultDeserializer<LineMetaData, LineRecord> createResultSetDeserializer() {
- return new TextResultDeserializer();
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Compaction.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Compaction.java
deleted file mode 100644
index abfbae9c99..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Compaction.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script;
-
-import org.apache.linkis.storage.script.compaction.PYScriptCompaction;
-import org.apache.linkis.storage.script.compaction.QLScriptCompaction;
-import org.apache.linkis.storage.script.compaction.ScalaScriptCompaction;
-import org.apache.linkis.storage.script.compaction.ShellScriptCompaction;
-
-public interface Compaction {
- String prefixConf();
-
- String prefix();
-
- boolean belongTo(String suffix);
-
- String compact(Variable variable);
-
- public static Compaction[] listCompactions() {
- return new Compaction[] {
- new PYScriptCompaction(),
- new QLScriptCompaction(),
- new ScalaScriptCompaction(),
- new ShellScriptCompaction()
- };
- }
-
- String getAnnotationSymbol();
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Parser.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Parser.java
deleted file mode 100644
index ee1ec19016..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Parser.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script;
-
-public interface Parser {
- String prefixConf();
-
- String prefix();
-
- boolean belongTo(String suffix);
-
- Variable parse(String line);
-
- String getAnnotationSymbol();
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ParserFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ParserFactory.java
deleted file mode 100644
index 58edf3dfe9..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ParserFactory.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script;
-
-import org.apache.linkis.storage.script.parser.PYScriptParser;
-import org.apache.linkis.storage.script.parser.QLScriptParser;
-import org.apache.linkis.storage.script.parser.ScalaScriptParser;
-import org.apache.linkis.storage.script.parser.ShellScriptParser;
-
-public class ParserFactory {
- public static Parser[] listParsers() {
- return new Parser[] {
- new PYScriptParser(), new QLScriptParser(), new ScalaScriptParser(), new ShellScriptParser()
- };
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptFsReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptFsReader.java
deleted file mode 100644
index 00b79eb7e4..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptFsReader.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script;
-
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.common.io.FsReader;
-import org.apache.linkis.storage.script.reader.StorageScriptFsReader;
-
-import java.io.InputStream;
-
-public abstract class ScriptFsReader extends FsReader {
-
- protected FsPath path;
- protected String charset;
-
- public ScriptFsReader(FsPath path, String charset) {
- this.path = path;
- this.charset = charset;
- }
-
- public static ScriptFsReader getScriptFsReader(
- FsPath path, String charset, InputStream inputStream) {
- return new StorageScriptFsReader(path, charset, inputStream);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptFsWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptFsWriter.java
deleted file mode 100644
index 835e1bfb21..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptFsWriter.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script;
-
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.common.io.FsWriter;
-import org.apache.linkis.storage.script.writer.StorageScriptFsWriter;
-
-import java.io.InputStream;
-import java.io.OutputStream;
-
-public abstract class ScriptFsWriter extends FsWriter {
- FsPath path;
- String charset;
-
- public abstract InputStream getInputStream();
-
- public static ScriptFsWriter getScriptFsWriter(
- FsPath path, String charset, OutputStream outputStream) {
- return new StorageScriptFsWriter(path, charset, outputStream);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptMetaData.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptMetaData.java
deleted file mode 100644
index 28999aae75..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptMetaData.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script;
-
-import org.apache.linkis.common.io.MetaData;
-
-public class ScriptMetaData implements MetaData {
- private Variable[] variables;
-
- public ScriptMetaData(Variable[] variables) {
- this.variables = variables;
- }
-
- @Override
- public MetaData cloneMeta() {
- return new ScriptMetaData(variables.clone());
- }
-
- public Variable[] getMetaData() {
- return variables;
- }
-
- public void setMetaData(Variable[] variables) {
- this.variables = variables;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptRecord.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptRecord.java
deleted file mode 100644
index c948b9558c..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptRecord.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script;
-
-import org.apache.linkis.storage.LineRecord;
-
-public class ScriptRecord extends LineRecord {
- public ScriptRecord(String line) {
- super(line);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Variable.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Variable.java
deleted file mode 100644
index 8c726671ba..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Variable.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script;
-
-public class Variable {
- protected String sortParent;
- protected String sort;
- protected String key;
- protected String value;
-
- public Variable(String sortParent, String sort, String key, String value) {
- this.sortParent = sortParent;
- this.sort = sort;
- this.key = key;
- this.value = value;
- }
-
- public String getSortParent() {
- return sortParent;
- }
-
- public String getSort() {
- return sort;
- }
-
- public String getKey() {
- return key;
- }
-
- public String getValue() {
- return value;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/VariableParser.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/VariableParser.java
deleted file mode 100644
index cd10746457..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/VariableParser.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script;
-
-import java.util.*;
-
-public class VariableParser {
-
- public static final String CONFIGURATION = "configuration";
- public static final String VARIABLE = "variable";
- public static final String RUNTIME = "runtime";
- public static final String STARTUP = "startup";
- public static final String SPECIAL = "special";
-
- public static Variable[] getVariables(Map<String, Object> params) {
- List<Variable> variables = new ArrayList<>();
- Map<String, Object> variableMap =
- (Map<String, Object>) params.getOrDefault(VARIABLE, new HashMap<String, Object>());
- for (Map.Entry<String, Object> entry : variableMap.entrySet()) {
- variables.add(new Variable(VARIABLE, null, entry.getKey(), entry.getValue().toString()));
- }
-
- Map<String, Object> configurationMap =
- (Map<String, Object>) params.getOrDefault(CONFIGURATION, new HashMap<String, Object>());
- for (Map.Entry<String, Object> entry : configurationMap.entrySet()) {
- Map<String, Object> subMap = (Map<String, Object>) entry.getValue();
- for (Map.Entry<String, Object> subEntry : subMap.entrySet()) {
- if (!isContextIDINFO(subEntry.getKey())) {
- Object value = subEntry.getValue();
- if (value instanceof Map) {
- Map<String, Object> innerMap = (Map<String, Object>) value;
- for (Map.Entry<String, Object> innerEntry : innerMap.entrySet()) {
- if (!isContextIDINFO(innerEntry.getKey())) {
- variables.add(
- new Variable(
- entry.getKey(),
- subEntry.getKey(),
- innerEntry.getKey(),
- innerEntry.getValue().toString()));
- }
- }
- } else {
- if (value == null) {
- variables.add(new Variable(CONFIGURATION, entry.getKey(), subEntry.getKey(), ""));
- } else {
- variables.add(
- new Variable(CONFIGURATION, entry.getKey(), subEntry.getKey(), value.toString()));
- }
- }
- }
- }
- }
- return variables.toArray(new Variable[variables.size()]);
- }
-
- private static boolean isContextIDINFO(String key) {
- return "contextID".equalsIgnoreCase(key) || "nodeName".equalsIgnoreCase(key);
- }
-
- public static Map<String, Object> getMap(Variable[] variables) {
- Map<String, String> variableKey2Value = new HashMap<>();
- Map<String, Object> confs = new HashMap<>();
-
- Arrays.stream(variables)
- .filter(variable -> variable.sort == null)
- .forEach(v -> variableKey2Value.put(v.key, v.value));
-
- Arrays.stream(variables)
- .filter(variable -> variable.sort != null)
- .forEach(
- v -> {
- switch (v.getSort()) {
- case STARTUP:
- case RUNTIME:
- case SPECIAL:
- if (!confs.containsKey(v.getSort())) {
- confs.put(v.getSort(), createMap(v));
- } else {
- Map<String, Object> subMap = (Map<String, Object>) confs.get(v.getSort());
- subMap.put(v.getKey(), v.getValue());
- }
- break;
- default:
- if (!confs.containsKey(v.getSortParent())) {
- Map<String, Object> subMap = new HashMap<>();
- subMap.put(v.getSort(), createMap(v));
- confs.put(v.getSortParent(), subMap);
- } else {
- Map<String, Object> subMap = (Map<String, Object>) confs.get(v.getSortParent());
- if (!subMap.containsKey(v.getSort())) {
- subMap.put(v.getSort(), createMap(v));
- } else {
- Map<String, Object> innerMap = (Map<String, Object>) subMap.get(v.getSort());
- innerMap.put(v.getKey(), v.getValue());
- }
- }
- break;
- }
- });
-
- Map<String, Object> params = new HashMap<>();
- if (!variableKey2Value.isEmpty()) {
- params.put(VARIABLE, variableKey2Value);
- }
- if (!confs.isEmpty()) {
- params.put(CONFIGURATION, confs);
- }
- return params;
- }
-
- private static Map<String, Object> createMap(Variable variable) {
- Map<String, Object> map = new HashMap<>();
- map.put(variable.getKey(), variable.getValue());
- return map;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.java
deleted file mode 100644
index f8e01a682c..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script.compaction;
-
-import org.apache.linkis.storage.script.Compaction;
-import org.apache.linkis.storage.script.Variable;
-
-public abstract class CommonScriptCompaction implements Compaction {
-
- @Override
- public String compact(Variable variable) {
- switch (variable.getSortParent()) {
- case "variable":
- return prefix() + " " + variable.getKey() + "=" + variable.getValue();
- default:
- return prefixConf()
- + " "
- + variable.getSortParent()
- + " "
- + variable.getSort()
- + " "
- + variable.getKey()
- + "="
- + variable.getValue();
- }
- }
-
- @Override
- public String getAnnotationSymbol() {
- return prefix().split("@")[0];
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/PYScriptCompaction.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/PYScriptCompaction.java
deleted file mode 100644
index 4022e7d46d..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/PYScriptCompaction.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script.compaction;
-
-import org.apache.linkis.common.utils.CodeAndRunTypeUtils;
-
-public class PYScriptCompaction extends CommonScriptCompaction {
-
- private static final PYScriptCompaction pYScriptCompaction = new PYScriptCompaction();
-
- public static CommonScriptCompaction apply() {
- return pYScriptCompaction;
- }
-
- @Override
- public boolean belongTo(String suffix) {
- return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot(
- suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_PYTHON());
- }
-
- @Override
- public String prefix() {
- return "#@set";
- }
-
- @Override
- public String prefixConf() {
- return "#conf@set";
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/QLScriptCompaction.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/QLScriptCompaction.java
deleted file mode 100644
index 97fc29efa0..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/QLScriptCompaction.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script.compaction;
-
-import org.apache.linkis.common.utils.CodeAndRunTypeUtils;
-
-public class QLScriptCompaction extends CommonScriptCompaction {
-
- private static final QLScriptCompaction qLScriptCompaction = new QLScriptCompaction();
-
- public static CommonScriptCompaction apply() {
- return qLScriptCompaction;
- }
-
- @Override
- public boolean belongTo(String suffix) {
- return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot(
- suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL());
- }
-
- @Override
- public String prefix() {
- return "--@set";
- }
-
- @Override
- public String prefixConf() {
- return "--conf@set";
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/ScalaScriptCompaction.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/ScalaScriptCompaction.java
deleted file mode 100644
index 18d5d2f531..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/ScalaScriptCompaction.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script.compaction;
-
-import org.apache.linkis.common.utils.CodeAndRunTypeUtils;
-
-public class ScalaScriptCompaction extends CommonScriptCompaction {
-
- private static final ScalaScriptCompaction compaction = new ScalaScriptCompaction();
-
- public static CommonScriptCompaction apply() {
- return compaction;
- }
-
- @Override
- public String prefix() {
- return "//@set";
- }
-
- @Override
- public boolean belongTo(String suffix) {
- return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot(
- suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_SCALA());
- }
-
- @Override
- public String prefixConf() {
- return "//conf@set";
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/ShellScriptCompaction.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/ShellScriptCompaction.java
deleted file mode 100644
index 085815b559..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/ShellScriptCompaction.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script.compaction;
-
-import org.apache.linkis.common.utils.CodeAndRunTypeUtils;
-
-public class ShellScriptCompaction extends CommonScriptCompaction {
-
- private static final ShellScriptCompaction shellScriptCompaction = new ShellScriptCompaction();
-
- public static CommonScriptCompaction apply() {
- return shellScriptCompaction;
- }
-
- @Override
- public String prefix() {
- return "#@set";
- }
-
- @Override
- public String prefixConf() {
- return "#conf@set";
- }
-
- @Override
- public boolean belongTo(String suffix) {
- return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot(
- suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_SHELL());
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/CommonScriptParser.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/CommonScriptParser.java
deleted file mode 100644
index 0d7e9f58e1..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/CommonScriptParser.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script.parser;
-
-import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary;
-import org.apache.linkis.storage.exception.StorageWarnException;
-import org.apache.linkis.storage.script.Parser;
-import org.apache.linkis.storage.script.Variable;
-import org.apache.linkis.storage.script.VariableParser;
-
-import java.util.Arrays;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public abstract class CommonScriptParser implements Parser {
-
- @Override
- public Variable parse(String line) {
- String variableReg = "\\s*" + prefix() + "\\s*(.+)\\s*" + "=" + "\\s*(.+)\\s*";
- Pattern pattern = Pattern.compile(variableReg);
- Matcher matcher = pattern.matcher(line);
- if (matcher.matches()) {
- String key = matcher.group(1).trim();
- String value = matcher.group(2).trim();
- return new Variable(VariableParser.VARIABLE, null, key, value);
-
- } else {
- String[] splitLine = line.split("=");
- if (splitLine.length != 2) {
- throw new StorageWarnException(
- LinkisStorageErrorCodeSummary.INVALID_CUSTOM_PARAMETER.getErrorCode(),
- LinkisStorageErrorCodeSummary.INVALID_CUSTOM_PARAMETER.getErrorDesc());
- }
- String[] subSplit =
- Arrays.stream(splitLine[0].split(" "))
- .filter(str -> !"".equals(str))
- .toArray(String[]::new);
- if (subSplit.length != 4) {
- throw new StorageWarnException(
- LinkisStorageErrorCodeSummary.INVALID_CUSTOM_PARAMETER.getErrorCode(),
- LinkisStorageErrorCodeSummary.INVALID_CUSTOM_PARAMETER.getErrorDesc());
- }
- if (!subSplit[0].trim().equals(prefixConf())) {
- throw new StorageWarnException(
- LinkisStorageErrorCodeSummary.INVALID_CUSTOM_PARAMETER.getErrorCode(),
- LinkisStorageErrorCodeSummary.INVALID_CUSTOM_PARAMETER.getErrorDesc());
- }
- String sortParent = subSplit[1].trim();
- String sort = subSplit[2].trim();
- String key = subSplit[3].trim();
- String value = splitLine[1].trim();
- return new Variable(sortParent, sort, key, value);
- }
- }
-
- @Override
- public String getAnnotationSymbol() {
- return prefix().split("@")[0];
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/PYScriptParser.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/PYScriptParser.java
deleted file mode 100644
index 3d5edcfac7..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/PYScriptParser.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script.parser;
-
-import org.apache.linkis.common.utils.CodeAndRunTypeUtils;
-
-public class PYScriptParser extends CommonScriptParser {
-
- private static final PYScriptParser pYScriptParser = new PYScriptParser();
-
- public static CommonScriptParser apply() {
- return pYScriptParser;
- }
-
- @Override
- public String prefix() {
- return "#@set";
- }
-
- @Override
- public boolean belongTo(String suffix) {
- return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot(
- suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_PYTHON());
- }
-
- @Override
- public String prefixConf() {
- return "#conf@set";
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/QLScriptParser.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/QLScriptParser.java
deleted file mode 100644
index 806a84760f..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/QLScriptParser.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script.parser;
-
-import org.apache.linkis.common.utils.CodeAndRunTypeUtils;
-
-public class QLScriptParser extends CommonScriptParser {
-
- private static final QLScriptParser qLScriptParser = new QLScriptParser();
-
- public static CommonScriptParser apply() {
- return qLScriptParser;
- }
-
- @Override
- public String prefix() {
- return "--@set";
- }
-
- @Override
- public boolean belongTo(String suffix) {
- return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot(
- suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL());
- }
-
- @Override
- public String prefixConf() {
- return "--conf@set";
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/ScalaScriptParser.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/ScalaScriptParser.java
deleted file mode 100644
index 982538d8e8..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/ScalaScriptParser.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script.parser;
-
-import org.apache.linkis.common.utils.CodeAndRunTypeUtils;
-
-public class ScalaScriptParser extends CommonScriptParser {
-
- private static final ScalaScriptParser scalaScriptParser = new ScalaScriptParser();
-
- public static CommonScriptParser apply() {
- return scalaScriptParser;
- }
-
- @Override
- public String prefix() {
- return "//@set";
- }
-
- @Override
- public boolean belongTo(String suffix) {
- return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot(
- suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_SCALA());
- }
-
- @Override
- public String prefixConf() {
- return "//conf@set";
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/ShellScriptParser.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/ShellScriptParser.java
deleted file mode 100644
index e1eebe3e58..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/ShellScriptParser.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script.parser;
-
-import org.apache.linkis.common.utils.CodeAndRunTypeUtils;
-
-public class ShellScriptParser extends CommonScriptParser {
-
- private static final ShellScriptParser shellScriptParser = new ShellScriptParser();
-
- public static CommonScriptParser create() {
- return shellScriptParser;
- }
-
- @Override
- public String prefix() {
- return "#@set";
- }
-
- @Override
- public boolean belongTo(String suffix) {
- return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot(
- suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_SHELL());
- }
-
- @Override
- public String prefixConf() {
- return "#conf@set";
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/reader/StorageScriptFsReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/reader/StorageScriptFsReader.java
deleted file mode 100644
index 1f862fa6b1..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/reader/StorageScriptFsReader.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script.reader;
-
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.storage.script.*;
-import org.apache.linkis.storage.utils.StorageUtils;
-
-import org.apache.commons.io.IOUtils;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-public class StorageScriptFsReader extends ScriptFsReader {
- private final FsPath path;
- private final String charset;
- private final InputStream inputStream;
-
- private InputStreamReader inputStreamReader;
- private BufferedReader bufferedReader;
-
- private ScriptMetaData metadata;
-
- private List<Variable> variables = new ArrayList<>();
- private String lineText;
-
- public StorageScriptFsReader(FsPath path, String charset, InputStream inputStream) {
- super(path, charset);
- this.path = path;
- this.charset = charset;
- this.inputStream = inputStream;
- }
-
- @Override
- public Record getRecord() throws IOException {
- if (metadata == null) throw new IOException("Must read metadata first(必须先读取metadata)");
- ScriptRecord record = new ScriptRecord(lineText);
- lineText = bufferedReader.readLine();
- return record;
- }
-
- @Override
- public MetaData getMetaData() throws IOException {
- if (metadata == null) init();
- Parser parser = getScriptParser();
- lineText = bufferedReader.readLine();
- while (hasNext()
- && Objects.nonNull(parser)
- && isMetadata(lineText, parser.prefix(), parser.prefixConf())) {
- variables.add(parser.parse(lineText));
- lineText = bufferedReader.readLine();
- }
- metadata = new ScriptMetaData(variables.toArray(new Variable[0]));
- return metadata;
- }
-
- public void init() {
- inputStreamReader = new InputStreamReader(inputStream);
- bufferedReader = new BufferedReader(inputStreamReader);
- }
-
- @Override
- public int skip(int recordNum) throws IOException {
- if (recordNum < 0) return -1;
- if (metadata == null) getMetaData();
- try {
- return (int) bufferedReader.skip(recordNum);
- } catch (Throwable t) {
- return recordNum;
- }
- }
-
- @Override
- public long getPosition() throws IOException {
- return -1L;
- }
-
- @Override
- public boolean hasNext() throws IOException {
- return lineText != null;
- }
-
- @Override
- public long available() throws IOException {
- return inputStream != null ? inputStream.available() : 0L;
- }
-
- @Override
- public void close() throws IOException {
- IOUtils.closeQuietly(bufferedReader);
- IOUtils.closeQuietly(inputStreamReader);
- IOUtils.closeQuietly(inputStream);
- }
-
- /**
- * Determine if the read line is metadata(判断读的行是否是metadata)
- *
- * @param line
- * @return
- */
- public boolean isMetadata(String line, String prefix, String prefixConf) {
- String regex = "\\s*" + prefix + "\\s*(.+)\\s*=\\s*(.+)\\s*";
- if (line.matches(regex)) {
- return true;
- } else {
- String[] split = line.split("=");
- if (split.length != 2) {
- return false;
- }
- if (Stream.of(split[0].split(" ")).filter(str -> !"".equals(str)).count() != 4) {
- return false;
- }
-
- Optional<String> optional =
- Stream.of(split[0].split(" ")).filter(str -> !"".equals(str)).findFirst();
- if (optional.isPresent() && !optional.get().equals(prefixConf)) {
- return false;
- }
- return true;
- }
- }
-
- /**
- * get the script parser according to the path(根据文件路径 获取对应的script parser )
- *
- * @return Scripts Parser
- */
- public Parser getScriptParser() {
- List<Parser> parsers =
- Arrays.stream(ParserFactory.listParsers())
- .filter(p -> p.belongTo(StorageUtils.pathToSuffix(path.getPath())))
- .collect(Collectors.toList());
- if (parsers.size() > 0) {
- return parsers.get(0);
- } else {
- return null;
- }
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.java
deleted file mode 100644
index 84dd6abb83..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.script.writer;
-
-import org.apache.linkis.common.io.FsPath;
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.storage.LineRecord;
-import org.apache.linkis.storage.script.Compaction;
-import org.apache.linkis.storage.script.ScriptFsWriter;
-import org.apache.linkis.storage.script.ScriptMetaData;
-import org.apache.linkis.storage.script.Variable;
-import org.apache.linkis.storage.utils.StorageConfiguration;
-import org.apache.linkis.storage.utils.StorageUtils;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
-
-import java.io.*;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class StorageScriptFsWriter extends ScriptFsWriter {
- private static final Logger log = LoggerFactory.getLogger(StorageScriptFsWriter.class);
-
- private final FsPath path;
- private final String charset;
- private final OutputStream outputStream;
- private final StringBuilder stringBuilder = new StringBuilder();
-
- public StorageScriptFsWriter(FsPath path, String charset, OutputStream outputStream) {
- this.path = path;
- this.charset = charset;
- this.outputStream = outputStream;
- }
-
- @Override
- public void addMetaData(MetaData metaData) throws IOException {
- String suffix = StorageUtils.pathToSuffix(path.getPath());
- List<Compaction> compactions =
- Stream.of(Compaction.listCompactions())
- .filter(compaction -> compaction.belongTo(suffix))
- .collect(Collectors.toList());
- List<String> metadataLine = new ArrayList<>();
- if (!compactions.isEmpty()) {
- Variable[] metaData1 = ((ScriptMetaData) metaData).getMetaData();
- Stream.of(metaData1).map(compactions.get(0)::compact).forEach(metadataLine::add);
-
- // add annotition symbol
- if (metadataLine.size() > 0) {
- metadataLine.add(compactions.get(0).getAnnotationSymbol());
- }
- if (outputStream != null) {
- IOUtils.writeLines(metadataLine, "\n", outputStream, charset);
- } else {
- metadataLine.forEach(m -> stringBuilder.append(m).append("\n"));
- }
- }
- }
-
- @Override
- public void addRecord(Record record) throws IOException {
- LineRecord scriptRecord = (LineRecord) record;
- if (outputStream != null) {
- IOUtils.write(scriptRecord.getLine(), outputStream, charset);
- } else {
- stringBuilder.append(scriptRecord.getLine());
- }
- }
-
- @Override
- public void close() {
- IOUtils.closeQuietly(outputStream);
- }
-
- @Override
- public void flush() {
- if (outputStream instanceof HdfsDataOutputStream) {
- try {
- ((HdfsDataOutputStream) outputStream).hflush();
- } catch (IOException t) {
- log.warn("Error encountered when flush script", t);
- }
- } else if (outputStream != null) {
- try {
- outputStream.flush();
- } catch (IOException t) {
- log.warn("Error encountered when flush script", t);
- }
- }
- }
-
- @Override
- public InputStream getInputStream() {
- byte[] bytes = null;
- try {
- bytes =
- stringBuilder.toString().getBytes(StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue());
- } catch (UnsupportedEncodingException e) {
- log.warn("StorageScriptFsWriter getInputStream failed", e);
- }
- return new ByteArrayInputStream(bytes);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/AbstractFileSource.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/AbstractFileSource.java
deleted file mode 100644
index fc4e615b36..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/AbstractFileSource.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.source;
-
-import org.apache.linkis.common.io.FsWriter;
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.math3.util.Pair;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-public abstract class AbstractFileSource implements FileSource {
-
- private FileSplit[] fileSplits;
-
- public AbstractFileSource(FileSplit[] fileSplits) {
- this.fileSplits = fileSplits;
- }
-
- @Override
- public FileSource shuffle(Function<Record, Record> function) {
- Arrays.stream(fileSplits).forEach(fileSplit -> fileSplit.shuffler = function);
- return this;
- }
-
- @Override
- public FileSource page(int page, int pageSize) {
- Arrays.stream(fileSplits).forEach(fileSplit -> fileSplit.page(page, pageSize));
- return this;
- }
-
- @Override
- public FileSource addParams(Map<String, String> params) {
- Arrays.stream(fileSplits).forEach(fileSplit -> fileSplit.addParams(params));
- return this;
- }
-
- @Override
- public FileSource addParams(String key, String value) {
- Arrays.stream(fileSplits).forEach(fileSplit -> fileSplit.addParams(key, value));
- return this;
- }
-
- @Override
- public FileSplit[] getFileSplits() {
- return this.fileSplits;
- }
-
- @Override
- public Map<String, String> getParams() {
- return Arrays.stream(fileSplits)
- .map(FileSplit::getParams)
- .flatMap(map -> map.entrySet().stream())
- .collect(
- Collectors.toMap(
- Map.Entry::getKey, Map.Entry::getValue, (existingValue, newValue) -> newValue));
- }
-
- @Override
- public <K extends MetaData, V extends Record> void write(FsWriter<K, V> fsWriter) {
- Arrays.stream(fileSplits).forEach(fileSplit -> fileSplit.write(fsWriter));
- }
-
- @Override
- public void close() {
- Arrays.stream(fileSplits).forEach(IOUtils::closeQuietly);
- }
-
- @Override
- public Pair<Object, List<String[]>>[] collect() {
- return Arrays.stream(fileSplits).map(FileSplit::collect).toArray(Pair[]::new);
- }
-
- @Override
- public int getTotalLine() {
- return Arrays.stream(fileSplits).mapToInt(FileSplit::getTotalLine).sum();
- }
-
- @Override
- public String[] getTypes() {
- return Arrays.stream(fileSplits).map(FileSplit::getType).toArray(String[]::new);
- }
-
- @Override
- public Pair<Integer, Integer>[] getFileInfo(int needToCountRowNumber) {
- return Arrays.stream(fileSplits)
- .map(fileSplit -> fileSplit.getFileInfo(needToCountRowNumber))
- .toArray(Pair[]::new);
- }
-
- @Override
- public FileSource limitBytes(Long limitBytes) {
- Arrays.stream(fileSplits).forEach(fileSplit -> fileSplit.setLimitBytes(limitBytes));
- return this;
- }
-
- @Override
- public FileSource limitColumnLength(int limitColumnLength) {
- Arrays.stream(fileSplits)
- .forEach(fileSplit -> fileSplit.setLimitColumnLength(limitColumnLength));
- return this;
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java
deleted file mode 100644
index f7a5d3e512..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.source;
-
-import org.apache.linkis.common.io.*;
-import org.apache.linkis.common.io.resultset.ResultSet;
-import org.apache.linkis.common.io.resultset.ResultSetReader;
-import org.apache.linkis.storage.conf.LinkisStorageConf;
-import org.apache.linkis.storage.exception.StorageWarnException;
-import org.apache.linkis.storage.resultset.ResultSetFactory;
-import org.apache.linkis.storage.resultset.ResultSetReaderFactory;
-import org.apache.linkis.storage.script.ScriptFsReader;
-import org.apache.linkis.storage.utils.StorageConfiguration;
-
-import org.apache.commons.math3.util.Pair;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE;
-
-public interface FileSource extends Closeable {
- Logger logger = LoggerFactory.getLogger(FileSource.class);
-
- FileSource shuffle(Function<Record, Record> s);
-
- FileSource page(int page, int pageSize);
-
- Pair<Object, List<String[]>>[] collect();
-
- Pair<Integer, Integer>[] getFileInfo(int needToCountRowNumber);
-
- <K extends MetaData, V extends Record> void write(FsWriter<K, V> fsWriter);
-
- FileSource addParams(Map<String, String> params);
-
- FileSource addParams(String key, String value);
-
- Map<String, String> getParams();
-
- int getTotalLine();
-
- String[] getTypes();
-
- FileSplit[] getFileSplits();
-
- String[] fileType = LinkisStorageConf.getFileTypeArr();
- BiFunction<String, String, Boolean> suffixPredicate =
- (path, suffix) -> path.endsWith("." + suffix);
-
- static boolean isResultSet(String path) {
- return suffixPredicate.apply(path, LinkisStorageConf.DOLPHIN)
- || suffixPredicate.apply(path, LinkisStorageConf.PARQUET)
- || suffixPredicate.apply(path, LinkisStorageConf.ORC);
- }
-
- static boolean isResultSet(FsPath fsPath) {
- return isResultSet(fsPath.getPath());
- }
-
- FileSource limitBytes(Long limitBytes);
-
- FileSource limitColumnLength(int limitColumnLength);
-
- /**
- * Currently only supports table multi-result sets
- *
- * @param fsPaths
- * @param fs
- * @return
- */
- static FileSource create(FsPath[] fsPaths, Fs fs) {
- // Filter non-table result sets
- FileSplit[] fileSplits =
- Arrays.stream(fsPaths)
- .map(fsPath -> createResultSetFileSplit(fsPath, fs))
- .filter(FileSource::isTableResultSet)
- .toArray(FileSplit[]::new);
- return new ResultsetFileSource(fileSplits);
- }
-
- static boolean isTableResultSet(FileSplit fileSplit) {
- return fileSplit.type.equals(ResultSetFactory.TABLE_TYPE);
- }
-
- static boolean isTableResultSet(FileSource fileSource) {
- // Return true only if all splits are table result sets
- return Arrays.stream(fileSource.getFileSplits()).allMatch(FileSource::isTableResultSet);
- }
-
- static FileSource create(FsPath fsPath, Fs fs) {
- if (!canRead(fsPath.getPath())) {
- throw new StorageWarnException(
- UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode(), UNSUPPORTED_OPEN_FILE_TYPE.getErrorDesc());
- }
- if (isResultSet(fsPath)) {
- return new ResultsetFileSource(new FileSplit[] {createResultSetFileSplit(fsPath, fs)});
- } else {
- return new TextFileSource(new FileSplit[] {createTextFileSplit(fsPath, fs)});
- }
- }
-
- static FileSource create(FsPath fsPath, InputStream is) {
- if (!canRead(fsPath.getPath())) {
- throw new StorageWarnException(
- UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode(), UNSUPPORTED_OPEN_FILE_TYPE.getErrorDesc());
- }
- if (isResultSet(fsPath)) {
- return new ResultsetFileSource(new FileSplit[] {createResultSetFileSplit(fsPath, is)});
- } else {
- return new TextFileSource(new FileSplit[] {createTextFileSplit(fsPath, is)});
- }
- }
-
- static FileSplit createResultSetFileSplit(FsPath fsPath, InputStream is) {
- logger.info("try create result set file split with path:{}", fsPath.getPath());
- ResultSet resultset = ResultSetFactory.getInstance().getResultSetByPath(fsPath);
- ResultSetReader resultsetReader =
- ResultSetReaderFactory.getResultSetReader(resultset, is, fsPath);
- return new FileSplit(resultsetReader, resultset.resultSetType());
- }
-
- static FileSplit createResultSetFileSplit(FsPath fsPath, Fs fs) {
- ResultSet resultset = ResultSetFactory.getInstance().getResultSetByPath(fsPath, fs);
- ResultSetReader resultsetReader = null;
- try {
- resultsetReader =
- ResultSetReaderFactory.getResultSetReader(resultset, fs.read(fsPath), fsPath);
- } catch (IOException e) {
- logger.warn("FileSource createResultSetFileSplit failed", e);
- }
- return new FileSplit(resultsetReader, resultset.resultSetType());
- }
-
- static FileSplit createTextFileSplit(FsPath fsPath, InputStream is) {
- ScriptFsReader scriptFsReader =
- ScriptFsReader.getScriptFsReader(
- fsPath, StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue(), is);
- return new FileSplit(scriptFsReader);
- }
-
- static FileSplit createTextFileSplit(FsPath fsPath, Fs fs) {
- ScriptFsReader scriptFsReader = null;
- try {
- scriptFsReader =
- ScriptFsReader.getScriptFsReader(
- fsPath, StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue(), fs.read(fsPath));
- } catch (IOException e) {
- logger.warn("FileSource createTextFileSplit failed", e);
- }
- return new FileSplit(scriptFsReader);
- }
-
- static boolean canRead(String path) {
- return Arrays.stream(fileType).anyMatch(suffix -> path.endsWith("." + suffix));
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSplit.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSplit.java
deleted file mode 100644
index 3a6c05a54a..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSplit.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.source;
-
-import org.apache.linkis.common.io.FsReader;
-import org.apache.linkis.common.io.FsWriter;
-import org.apache.linkis.common.io.MetaData;
-import org.apache.linkis.common.io.Record;
-import org.apache.linkis.storage.LineMetaData;
-import org.apache.linkis.storage.LineRecord;
-import org.apache.linkis.storage.domain.Column;
-import org.apache.linkis.storage.domain.DataType;
-import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary;
-import org.apache.linkis.storage.exception.StorageWarnException;
-import org.apache.linkis.storage.resultset.table.TableMetaData;
-import org.apache.linkis.storage.resultset.table.TableRecord;
-import org.apache.linkis.storage.script.Parser;
-import org.apache.linkis.storage.script.ScriptMetaData;
-import org.apache.linkis.storage.script.Variable;
-import org.apache.linkis.storage.script.VariableParser;
-import org.apache.linkis.storage.script.reader.StorageScriptFsReader;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.math3.util.Pair;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class FileSplit implements Closeable {
- private static final Logger logger = LoggerFactory.getLogger(FileSplit.class);
-
- private FsReader<? extends MetaData, ? extends Record> fsReader;
- protected String type = "script/text";
- private int start = 0;
- private int end = -1;
- private int count = 0;
- private int totalLine = 0;
- protected Function<Record, Record> shuffler;
- private boolean pageTrigger = false;
- protected Map<String, String> params = new HashMap<>();
- private long limitBytes = 0L;
- private int limitColumnLength = 0;
-
- public FileSplit(FsReader<? extends MetaData, ? extends Record> fsReader) {
- this.fsReader = fsReader;
- }
-
- public FileSplit(FsReader<? extends MetaData, ? extends Record> fsReader, String type) {
- this.fsReader = fsReader;
- this.type = type;
- }
-
- public void page(int page, int pageSize) {
- if (!pageTrigger) {
- start = (page - 1) * pageSize;
- end = pageSize * page - 1;
- pageTrigger = true;
- }
- }
-
- public String getType() {
- return type;
- }
-
- public void addParams(Map<String, String> params) {
- this.params.putAll(params);
- }
-
- public void addParams(String key, String value) {
- this.params.put(key, value);
- }
-
- public Map<String, String> getParams() {
- return params;
- }
-
- public int getTotalLine() {
- return totalLine;
- }
-
- public void setLimitBytes(long limitBytes) {
- this.limitBytes = limitBytes;
- }
-
- public void setLimitColumnLength(int limitColumnLength) {
- this.limitColumnLength = limitColumnLength;
- }
-
- public <M> M whileLoop(Function<MetaData, M> metaDataFunction, Consumer<Record> recordConsumer) {
- M m = null;
- try {
- MetaData metaData = fsReader.getMetaData();
- m = metaDataFunction.apply(metaData);
- if (pageTrigger) {
- fsReader.skip(start);
- }
- count = start;
- boolean hasRemovedFlag = false;
- while (fsReader.hasNext() && ifContinueRead()) {
- Record record = fsReader.getRecord();
- boolean needRemoveFlag = false;
- if (!hasRemovedFlag && fsReader instanceof StorageScriptFsReader) {
- Parser parser = ((StorageScriptFsReader) fsReader).getScriptParser();
- Variable[] meta = ((ScriptMetaData) metaData).getMetaData();
- if (meta != null
- && meta.length > 0
- && parser != null
- && parser.getAnnotationSymbol().equals(record.toString())) {
- needRemoveFlag = true;
- hasRemovedFlag = true;
- }
- }
- if (!needRemoveFlag) {
- recordConsumer.accept(shuffler.apply(record));
- totalLine++;
- count++;
- }
- }
- } catch (IOException e) {
- logger.warn("FileSplit forEach failed", e);
- throw new StorageWarnException(
- LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode(),
- LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorMessage());
- }
- return m;
- }
-
- public void biConsumerWhileLoop(
- Consumer<MetaData> metaDataFunction, Consumer<Record> recordConsumer) {
- try {
- MetaData metaData = fsReader.getMetaData();
- metaDataFunction.accept(metaData);
- if (pageTrigger) {
- fsReader.skip(start);
- }
- count = start;
- boolean hasRemovedFlag = false;
- while (fsReader.hasNext() && ifContinueRead()) {
- Record record = fsReader.getRecord();
- boolean needRemoveFlag = false;
- if (!hasRemovedFlag && fsReader instanceof StorageScriptFsReader) {
- Parser parser = ((StorageScriptFsReader) fsReader).getScriptParser();
- Variable[] meta = ((ScriptMetaData) metaData).getMetaData();
- if (meta != null
- && meta.length > 0
- && parser != null
- && parser.getAnnotationSymbol().equals(record.toString())) {
- needRemoveFlag = true;
- hasRemovedFlag = true;
- }
- }
- if (!needRemoveFlag) {
- recordConsumer.accept(shuffler.apply(record));
- totalLine++;
- count++;
- }
- }
- } catch (IOException e) {
- logger.warn("FileSplit forEach failed", e);
- throw new StorageWarnException(
- LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode(),
- LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorMessage());
- }
- }
-
- public Pair<Integer, Integer> getFileInfo(int needToCountRowNumber) {
- int colNumber = 0;
- int rowNumber = 0;
- MetaData metaData = null;
- try {
- metaData = fsReader.getMetaData();
- colNumber =
- metaData instanceof TableMetaData ? ((TableMetaData) metaData).getColumns().length : 1;
- rowNumber =
- needToCountRowNumber == -1
- ? fsReader.skip(Integer.MAX_VALUE)
- : fsReader.skip(needToCountRowNumber);
- } catch (IOException e) {
- logger.warn("FileSplit getFileInfo failed", e);
- throw new StorageWarnException(
- LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode(),
- LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorMessage());
- }
- return new Pair<>(colNumber, rowNumber);
- }
-
- public <K extends MetaData, V extends Record> void write(FsWriter<K, V> fsWriter) {
- biConsumerWhileLoop(
- metaData -> {
- try {
- fsWriter.addMetaData(metaData);
- } catch (IOException e) {
- logger.warn("FileSplit addMetaData failed", e);
- throw new StorageWarnException(
- LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode(),
- LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorMessage());
- }
- },
- record -> {
- try {
- fsWriter.addRecord(record);
- } catch (IOException e) {
- logger.warn("FileSplit addRecord failed", e);
- throw new StorageWarnException(
- LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode(),
- LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorMessage());
- }
- });
- }
-
- public Pair<Object, List<String[]>> collect() {
- List<String[]> recordList = new ArrayList<>();
- final AtomicLong tmpBytes = new AtomicLong(0L);
- final AtomicBoolean overFlag = new AtomicBoolean(false);
- Object metaData =
- whileLoop(
- collectMetaData -> collectMetaData(collectMetaData),
- r -> {
- if (!overFlag.get()) {
- String[] arr = collectRecord(r);
- if (limitBytes > 0) {
- for (int i = 0; i < arr.length; i++) {
- tmpBytes.addAndGet(arr[i].getBytes().length);
- if (overFlag.get() || tmpBytes.get() > limitBytes) {
- overFlag.set(true);
- arr[i] = "";
- }
- }
- recordList.add(arr);
- } else {
- recordList.add(arr);
- }
- }
- });
- return new Pair<>(metaData, recordList);
- }
-
- public String[] collectRecord(Record record) {
- if (record instanceof TableRecord) {
- TableRecord tableRecord = (TableRecord) record;
- if (limitColumnLength > 0) {
- return Arrays.stream(tableRecord.row)
- .map(
- obj -> {
- String col = DataType.valueToString(obj);
- if (col.length() > limitColumnLength) {
- return col.substring(0, limitColumnLength);
- } else {
- return col;
- }
- })
- .toArray(String[]::new);
- }
- return Arrays.stream(tableRecord.row).map(DataType::valueToString).toArray(String[]::new);
- } else if (record instanceof LineRecord) {
- LineRecord lineRecord = (LineRecord) record;
- return new String[] {lineRecord.getLine()};
- } else {
- throw new IllegalArgumentException("Unknown record type");
- }
- }
-
- public Object collectMetaData(MetaData metaData) {
- if (metaData instanceof ScriptMetaData) {
- ScriptMetaData scriptMetaData = (ScriptMetaData) metaData;
- return VariableParser.getMap(scriptMetaData.getMetaData());
- } else if (metaData instanceof LineMetaData) {
- LineMetaData lineMetaData = (LineMetaData) metaData;
- return lineMetaData.getMetaData();
- } else if (metaData instanceof TableMetaData) {
- TableMetaData tableMetaData = (TableMetaData) metaData;
- return Arrays.stream(tableMetaData.getColumns())
- .map(this::columnToMap)
- .collect(Collectors.toList());
- } else {
- throw new IllegalArgumentException("Unknown metadata type");
- }
- }
-
- private Map<String, String> columnToMap(Column column) {
- Map<String, String> stringMap = new HashMap<>();
- stringMap.put("columnName", column.getColumnName());
- stringMap.put("comment", column.getComment());
- stringMap.put("dataType", column.getDataType().getTypeName());
- return stringMap;
- }
-
- public boolean ifContinueRead() {
- return !pageTrigger || count <= end;
- }
-
- public boolean ifStartRead() {
- return !pageTrigger || count >= start;
- }
-
- @Override
- public void close() {
- IOUtils.closeQuietly(fsReader);
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java
deleted file mode 100644
index 54fd64daad..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.source;
-
-import org.apache.linkis.storage.domain.Dolphin;
-import org.apache.linkis.storage.resultset.table.TableRecord;
-import org.apache.linkis.storage.utils.StorageUtils;
-
-import java.util.Arrays;
-
-public class ResultsetFileSource extends AbstractFileSource {
-
- public ResultsetFileSource(FileSplit[] fileSplits) {
- super(fileSplits);
- shuffle(
- record -> {
- if (record instanceof TableRecord) {
- TableRecord tableRecord = (TableRecord) record;
- String nullValue = getParams().getOrDefault("nullValue", "NULL");
- return new TableRecord(
- Arrays.stream(tableRecord.row)
- .map(
- r -> {
- if (r == null || r.equals("NULL")) {
- if (nullValue.equals(Dolphin.LINKIS_NULL)) {
- return r;
- } else {
- return nullValue;
- }
- } else if (r.equals("")) {
- String emptyValue = getParams().getOrDefault("nullValue", "");
- if (emptyValue.equals(Dolphin.LINKIS_NULL)) {
- return "";
- } else {
- return emptyValue;
- }
- } else if (r instanceof Double) {
- return StorageUtils.doubleToString((Double) r);
- } else {
- return r;
- }
- })
- .toArray());
- } else {
- return record;
- }
- });
- }
-}
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/TextFileSource.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/TextFileSource.java
deleted file mode 100644
index 7e5396bf74..0000000000
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/TextFileSource.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.storage.source;
-
-import org.apache.linkis.storage.LineRecord;
-import org.apache.linkis.storage.script.ScriptRecord;
-
-import org.apache.commons.math3.util.Pair;
-
... 94737 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org